diff --git "a/general_blip_test_llava_swift_A100.json" "b/general_blip_test_llava_swift_A100.json" new file mode 100644--- /dev/null +++ "b/general_blip_test_llava_swift_A100.json" @@ -0,0 +1,61819 @@ +[ + { + "query": "<\nPrevious Actions: Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.17, 0.7033]\", \"lift_point\": \"[0.17, 0.7033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.17, 0.7033]\", \"lift_point\": \"[0.17, 0.7033]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9046, 0.9253]\", \"lift_point\": \"[0.9046, 0.9253]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.17, 0.7033]\", \"lift_point\": \"[0.17, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9046, 0.9253]\", \"lift_point\": \"[0.9046, 0.9253]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8801, 0.5601]\", \"lift_point\": \"[0.8801, 0.5601]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.17, 0.7033]\", \"lift_point\": \"[0.17, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9046, 0.9253]\", \"lift_point\": \"[0.9046, 0.9253]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8801, 0.5601]\", \"lift_point\": \"[0.8801, 0.5601]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4973]\", \"lift_point\": \"[0.4166, 0.4973]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4973]\", \"lift_point\": \"[0.4166, 0.4973]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4973]\", \"lift_point\": \"[0.4166, 0.4973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3954, 0.4839]\", \"lift_point\": \"[0.3954, 0.4839]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4973]\", \"lift_point\": \"[0.4166, 0.4973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3954, 0.4839]\", \"lift_point\": \"[0.3954, 0.4839]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4973]\", \"lift_point\": \"[0.4166, 0.4973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3954, 0.4839]\", \"lift_point\": \"[0.3954, 0.4839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4973]\", \"lift_point\": \"[0.4166, 0.4973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3954, 0.4839]\", \"lift_point\": \"[0.3954, 0.4839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9645, 0.2741]\", \"lift_point\": \"[0.9645, 0.2741]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4488, 0.5619]\", \"lift_point\": \"[0.4488, 0.5619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4973]\", \"lift_point\": \"[0.4166, 0.4973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3954, 0.4839]\", \"lift_point\": \"[0.3954, 0.4839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.0549]\", \"lift_point\": \"[0.0818, 0.0549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9645, 0.2741]\", \"lift_point\": \"[0.9645, 0.2741]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/90_17.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Chile?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.7205]\", \"lift_point\": \"[0.7768, 0.7205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.3891]\", \"lift_point\": \"[0.0574, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.4832]\", \"lift_point\": \"[0.4163, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8182, 0.6398]\", \"lift_point\": \"[0.8182, 0.6398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.7309]\", \"lift_point\": \"[0.7942, 0.7309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.5194]\", \"lift_point\": \"[0.5042, 0.5194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3929, 0.6891]\", \"lift_point\": \"[0.3929, 0.6891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.869, 0.5517]\", \"lift_point\": \"[0.869, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3874, 0.6603]\", \"lift_point\": \"[0.3874, 0.6603]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3333, 0.9527]\", \"lift_point\": \"[0.3333, 0.9527]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3874, 0.6603]\", \"lift_point\": \"[0.3874, 0.6603]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5257, 0.9212]\", \"lift_point\": \"[0.5257, 0.9212]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3874, 0.6603]\", \"lift_point\": \"[0.3874, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5257, 0.9212]\", \"lift_point\": \"[0.5257, 0.9212]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5716, 0.8472]\", \"lift_point\": \"[0.5716, 0.8472]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4946, 0.357]\", \"lift_point\": \"[0.4946, 0.357]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3874, 0.6603]\", \"lift_point\": \"[0.3874, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5257, 0.9212]\", \"lift_point\": \"[0.5257, 0.9212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5716, 0.8472]\", \"lift_point\": \"[0.5716, 0.8472]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/224_48.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3801, 0.3268]\", \"lift_point\": \"[0.3801, 0.3268]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3801, 0.3268]\", \"lift_point\": \"[0.3801, 0.3268]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6006, 0.4735]\", \"lift_point\": \"[0.6006, 0.4735]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3801, 0.3268]\", \"lift_point\": \"[0.3801, 0.3268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6006, 0.4735]\", \"lift_point\": \"[0.6006, 0.4735]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.899]\", \"lift_point\": \"[0.2735, 0.899]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8313, 0.559]\", \"lift_point\": \"[0.8313, 0.559]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3801, 0.3268]\", \"lift_point\": \"[0.3801, 0.3268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6006, 0.4735]\", \"lift_point\": \"[0.6006, 0.4735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.899]\", \"lift_point\": \"[0.2735, 0.899]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/356_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/550_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8732, 0.5281]\", \"lift_point\": \"[0.8732, 0.5281]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/550_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8732, 0.5281]\", \"lift_point\": \"[0.8732, 0.5281]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in San Francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/550_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8732, 0.5281]\", \"lift_point\": \"[0.8732, 0.5281]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in San Francisco\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/550_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8732, 0.5281]\", \"lift_point\": \"[0.8732, 0.5281]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in San Francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/550_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8187, 0.6262]\", \"lift_point\": \"[0.8187, 0.6262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/756_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8187, 0.6262]\", \"lift_point\": \"[0.8187, 0.6262]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1809, 0.5707]\", \"lift_point\": \"[0.1809, 0.5707]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/756_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8187, 0.6262]\", \"lift_point\": \"[0.8187, 0.6262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1809, 0.5707]\", \"lift_point\": \"[0.1809, 0.5707]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1091, 0.9355]\", \"lift_point\": \"[0.1091, 0.9355]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/756_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8187, 0.6262]\", \"lift_point\": \"[0.8187, 0.6262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1809, 0.5707]\", \"lift_point\": \"[0.1809, 0.5707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1091, 0.9355]\", \"lift_point\": \"[0.1091, 0.9355]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/756_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8187, 0.6262]\", \"lift_point\": \"[0.8187, 0.6262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1809, 0.5707]\", \"lift_point\": \"[0.1809, 0.5707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1091, 0.9355]\", \"lift_point\": \"[0.1091, 0.9355]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Chicago?\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/756_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8187, 0.6262]\", \"lift_point\": \"[0.8187, 0.6262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1809, 0.5707]\", \"lift_point\": \"[0.1809, 0.5707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1091, 0.9355]\", \"lift_point\": \"[0.1091, 0.9355]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/756_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8723, 0.6865]\", \"lift_point\": \"[0.8723, 0.6865]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8723, 0.6865]\", \"lift_point\": \"[0.8723, 0.6865]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.6451]\", \"lift_point\": \"[0.7749, 0.6451]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8723, 0.6865]\", \"lift_point\": \"[0.8723, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.6451]\", \"lift_point\": \"[0.7749, 0.6451]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.5617]\", \"lift_point\": \"[0.1698, 0.5617]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8723, 0.6865]\", \"lift_point\": \"[0.8723, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.6451]\", \"lift_point\": \"[0.7749, 0.6451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.5617]\", \"lift_point\": \"[0.1698, 0.5617]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9241]\", \"lift_point\": \"[0.113, 0.9241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8723, 0.6865]\", \"lift_point\": \"[0.8723, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.6451]\", \"lift_point\": \"[0.7749, 0.6451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.5617]\", \"lift_point\": \"[0.1698, 0.5617]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9241]\", \"lift_point\": \"[0.113, 0.9241]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Vietnam?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8723, 0.6865]\", \"lift_point\": \"[0.8723, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.6451]\", \"lift_point\": \"[0.7749, 0.6451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.5617]\", \"lift_point\": \"[0.1698, 0.5617]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9241]\", \"lift_point\": \"[0.113, 0.9241]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Vietnam?\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.877, 0.6724]\", \"lift_point\": \"[0.877, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8723, 0.6865]\", \"lift_point\": \"[0.8723, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.6451]\", \"lift_point\": \"[0.7749, 0.6451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.5617]\", \"lift_point\": \"[0.1698, 0.5617]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9241]\", \"lift_point\": \"[0.113, 0.9241]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Vietnam?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/804_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5363, 0.6639]\", \"lift_point\": \"[0.5363, 0.6639]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/821_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5363, 0.6639]\", \"lift_point\": \"[0.5363, 0.6639]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6403, 0.4089]\", \"lift_point\": \"[0.6403, 0.4089]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/821_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5363, 0.6639]\", \"lift_point\": \"[0.5363, 0.6639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6403, 0.4089]\", \"lift_point\": \"[0.6403, 0.4089]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/821_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7926, 0.5541]\", \"lift_point\": \"[0.7926, 0.5541]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7926, 0.5541]\", \"lift_point\": \"[0.7926, 0.5541]\", \"typed_text\": \"\" Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1718, 0.525]\", \"lift_point\": \"[0.1718, 0.525]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7926, 0.5541]\", \"lift_point\": \"[0.7926, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1718, 0.525]\", \"lift_point\": \"[0.1718, 0.525]\", \"typed_text\": \"\" Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1175, 0.9367]\", \"lift_point\": \"[0.1175, 0.9367]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7926, 0.5541]\", \"lift_point\": \"[0.7926, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1718, 0.525]\", \"lift_point\": \"[0.1718, 0.525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1175, 0.9367]\", \"lift_point\": \"[0.1175, 0.9367]\", \"typed_text\": \"\" Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Japan?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7926, 0.5541]\", \"lift_point\": \"[0.7926, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1718, 0.525]\", \"lift_point\": \"[0.1718, 0.525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1175, 0.9367]\", \"lift_point\": \"[0.1175, 0.9367]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Japan?\" Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7783, 0.6389]\", \"lift_point\": \"[0.7783, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7926, 0.5541]\", \"lift_point\": \"[0.7926, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1718, 0.525]\", \"lift_point\": \"[0.1718, 0.525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1175, 0.9367]\", \"lift_point\": \"[0.1175, 0.9367]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Japan?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Japan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/844_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6361]\", \"lift_point\": \"[0.7948, 0.6361]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/858_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6361]\", \"lift_point\": \"[0.7948, 0.6361]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1631, 0.7936]\", \"lift_point\": \"[0.1631, 0.7936]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/858_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6361]\", \"lift_point\": \"[0.7948, 0.6361]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1631, 0.7936]\", \"lift_point\": \"[0.1631, 0.7936]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.9399]\", \"lift_point\": \"[0.1071, 0.9399]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/858_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6361]\", \"lift_point\": \"[0.7948, 0.6361]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1631, 0.7936]\", \"lift_point\": \"[0.1631, 0.7936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.9399]\", \"lift_point\": \"[0.1071, 0.9399]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Seoul\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/858_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6361]\", \"lift_point\": \"[0.7948, 0.6361]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1631, 0.7936]\", \"lift_point\": \"[0.1631, 0.7936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.9399]\", \"lift_point\": \"[0.1071, 0.9399]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Seoul\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/858_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6361]\", \"lift_point\": \"[0.7948, 0.6361]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1631, 0.7936]\", \"lift_point\": \"[0.1631, 0.7936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.9399]\", \"lift_point\": \"[0.1071, 0.9399]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Seoul\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/858_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.529, 0.9613]\", \"lift_point\": \"[0.529, 0.9613]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.529, 0.9613]\", \"lift_point\": \"[0.529, 0.9613]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1668, 0.5882]\", \"lift_point\": \"[0.1668, 0.5882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.529, 0.9613]\", \"lift_point\": \"[0.529, 0.9613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1668, 0.5882]\", \"lift_point\": \"[0.1668, 0.5882]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1, 0.952]\", \"lift_point\": \"[0.1, 0.952]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.529, 0.9613]\", \"lift_point\": \"[0.529, 0.9613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1668, 0.5882]\", \"lift_point\": \"[0.1668, 0.5882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1, 0.952]\", \"lift_point\": \"[0.1, 0.952]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8275, 0.6317]\", \"lift_point\": \"[0.8275, 0.6317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.529, 0.9613]\", \"lift_point\": \"[0.529, 0.9613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1668, 0.5882]\", \"lift_point\": \"[0.1668, 0.5882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1, 0.952]\", \"lift_point\": \"[0.1, 0.952]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.529, 0.9613]\", \"lift_point\": \"[0.529, 0.9613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1668, 0.5882]\", \"lift_point\": \"[0.1668, 0.5882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1, 0.952]\", \"lift_point\": \"[0.1, 0.952]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8381, 0.2983]\", \"lift_point\": \"[0.8381, 0.2983]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7947, 0.6208]\", \"lift_point\": \"[0.7947, 0.6208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.529, 0.9613]\", \"lift_point\": \"[0.529, 0.9613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1668, 0.5882]\", \"lift_point\": \"[0.1668, 0.5882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1, 0.952]\", \"lift_point\": \"[0.1, 0.952]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8381, 0.2983]\", \"lift_point\": \"[0.8381, 0.2983]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/876_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8144, 0.5933]\", \"lift_point\": \"[0.8144, 0.5933]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8144, 0.5933]\", \"lift_point\": \"[0.8144, 0.5933]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6457]\", \"lift_point\": \"[0.8079, 0.6457]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8144, 0.5933]\", \"lift_point\": \"[0.8144, 0.5933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6457]\", \"lift_point\": \"[0.8079, 0.6457]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1625, 0.7341]\", \"lift_point\": \"[0.1625, 0.7341]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8144, 0.5933]\", \"lift_point\": \"[0.8144, 0.5933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6457]\", \"lift_point\": \"[0.8079, 0.6457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1625, 0.7341]\", \"lift_point\": \"[0.1625, 0.7341]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.9331]\", \"lift_point\": \"[0.1134, 0.9331]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8144, 0.5933]\", \"lift_point\": \"[0.8144, 0.5933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6457]\", \"lift_point\": \"[0.8079, 0.6457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1625, 0.7341]\", \"lift_point\": \"[0.1625, 0.7341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.9331]\", \"lift_point\": \"[0.1134, 0.9331]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Nordstrom?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8144, 0.5933]\", \"lift_point\": \"[0.8144, 0.5933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6457]\", \"lift_point\": \"[0.8079, 0.6457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1625, 0.7341]\", \"lift_point\": \"[0.1625, 0.7341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.9331]\", \"lift_point\": \"[0.1134, 0.9331]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Nordstrom?\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8144, 0.5933]\", \"lift_point\": \"[0.8144, 0.5933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6457]\", \"lift_point\": \"[0.8079, 0.6457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1625, 0.7341]\", \"lift_point\": \"[0.1625, 0.7341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.9331]\", \"lift_point\": \"[0.1134, 0.9331]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Nordstrom?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1008_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8153, 0.6201]\", \"lift_point\": \"[0.8153, 0.6201]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8153, 0.6201]\", \"lift_point\": \"[0.8153, 0.6201]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2286, 0.0469]\", \"lift_point\": \"[0.2286, 0.0469]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8153, 0.6201]\", \"lift_point\": \"[0.8153, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2286, 0.0469]\", \"lift_point\": \"[0.2286, 0.0469]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1685, 0.5766]\", \"lift_point\": \"[0.1685, 0.5766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8153, 0.6201]\", \"lift_point\": \"[0.8153, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2286, 0.0469]\", \"lift_point\": \"[0.2286, 0.0469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1685, 0.5766]\", \"lift_point\": \"[0.1685, 0.5766]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1106, 0.9401]\", \"lift_point\": \"[0.1106, 0.9401]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8153, 0.6201]\", \"lift_point\": \"[0.8153, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2286, 0.0469]\", \"lift_point\": \"[0.2286, 0.0469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1685, 0.5766]\", \"lift_point\": \"[0.1685, 0.5766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1106, 0.9401]\", \"lift_point\": \"[0.1106, 0.9401]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in San Francisco?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8153, 0.6201]\", \"lift_point\": \"[0.8153, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2286, 0.0469]\", \"lift_point\": \"[0.2286, 0.0469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1685, 0.5766]\", \"lift_point\": \"[0.1685, 0.5766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1106, 0.9401]\", \"lift_point\": \"[0.1106, 0.9401]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in San Francisco?\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.6898]\", \"lift_point\": \"[0.5409, 0.6898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8153, 0.6201]\", \"lift_point\": \"[0.8153, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2286, 0.0469]\", \"lift_point\": \"[0.2286, 0.0469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1685, 0.5766]\", \"lift_point\": \"[0.1685, 0.5766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1106, 0.9401]\", \"lift_point\": \"[0.1106, 0.9401]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in San Francisco?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1045_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1057_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1057_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.8526]\", \"lift_point\": \"[0.0528, 0.8526]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1057_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.8526]\", \"lift_point\": \"[0.0528, 0.8526]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.163, 0.5951]\", \"lift_point\": \"[0.163, 0.5951]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1057_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8044, 0.6462]\", \"lift_point\": \"[0.8044, 0.6462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.8526]\", \"lift_point\": \"[0.0528, 0.8526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.163, 0.5951]\", \"lift_point\": \"[0.163, 0.5951]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1057_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1095_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1095_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1742, 0.6779]\", \"lift_point\": \"[0.1742, 0.6779]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1095_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1742, 0.6779]\", \"lift_point\": \"[0.1742, 0.6779]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1824, 0.6596]\", \"lift_point\": \"[0.1824, 0.6596]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1095_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8103, 0.5941]\", \"lift_point\": \"[0.8103, 0.5941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1742, 0.6779]\", \"lift_point\": \"[0.1742, 0.6779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1824, 0.6596]\", \"lift_point\": \"[0.1824, 0.6596]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1095_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is the sun?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.7015]\", \"lift_point\": \"[0.789, 0.7015]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1205_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.7015]\", \"lift_point\": \"[0.789, 0.7015]\", \"typed_text\": \"\" Goal: How big is the sun?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2867]\", \"lift_point\": \"[0.0518, 0.2867]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1205_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.7015]\", \"lift_point\": \"[0.789, 0.7015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2867]\", \"lift_point\": \"[0.0518, 0.2867]\", \"typed_text\": \"\" Goal: How big is the sun?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the sun?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1205_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.7015]\", \"lift_point\": \"[0.789, 0.7015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2867]\", \"lift_point\": \"[0.0518, 0.2867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the sun?\" Goal: How big is the sun?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.4175]\", \"lift_point\": \"[0.1291, 0.4175]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1205_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.7015]\", \"lift_point\": \"[0.789, 0.7015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2867]\", \"lift_point\": \"[0.0518, 0.2867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the sun?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.4175]\", \"lift_point\": \"[0.1291, 0.4175]\", \"typed_text\": \"\" Goal: How big is the sun?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1205_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7852, 0.6943]\", \"lift_point\": \"[0.7852, 0.6943]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1267_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7852, 0.6943]\", \"lift_point\": \"[0.7852, 0.6943]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.4661]\", \"lift_point\": \"[0.0693, 0.4661]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1267_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7852, 0.6943]\", \"lift_point\": \"[0.7852, 0.6943]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.4661]\", \"lift_point\": \"[0.0693, 0.4661]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rate against the Chinese Yuan?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1267_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7852, 0.6943]\", \"lift_point\": \"[0.7852, 0.6943]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.4661]\", \"lift_point\": \"[0.0693, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rate against the Chinese Yuan?\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.3515]\", \"lift_point\": \"[0.0616, 0.3515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1267_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7852, 0.6943]\", \"lift_point\": \"[0.7852, 0.6943]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.4661]\", \"lift_point\": \"[0.0693, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rate against the Chinese Yuan?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.3515]\", \"lift_point\": \"[0.0616, 0.3515]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1402, 0.5243]\", \"lift_point\": \"[0.1402, 0.5243]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1267_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7852, 0.6943]\", \"lift_point\": \"[0.7852, 0.6943]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.4661]\", \"lift_point\": \"[0.0693, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rate against the Chinese Yuan?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.3515]\", \"lift_point\": \"[0.0616, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1402, 0.5243]\", \"lift_point\": \"[0.1402, 0.5243]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1267_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6605]\", \"lift_point\": \"[0.7839, 0.6605]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6605]\", \"lift_point\": \"[0.7839, 0.6605]\", \"typed_text\": \"\" Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4753]\", \"lift_point\": \"[0.0562, 0.4753]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6605]\", \"lift_point\": \"[0.7839, 0.6605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4753]\", \"lift_point\": \"[0.0562, 0.4753]\", \"typed_text\": \"\" Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Uruguay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6605]\", \"lift_point\": \"[0.7839, 0.6605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4753]\", \"lift_point\": \"[0.0562, 0.4753]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Uruguay?\" Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6203]\", \"lift_point\": \"[0.0533, 0.6203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6605]\", \"lift_point\": \"[0.7839, 0.6605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4753]\", \"lift_point\": \"[0.0562, 0.4753]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Uruguay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6203]\", \"lift_point\": \"[0.0533, 0.6203]\", \"typed_text\": \"\" Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1402, 0.6363]\", \"lift_point\": \"[0.1402, 0.6363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6605]\", \"lift_point\": \"[0.7839, 0.6605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4753]\", \"lift_point\": \"[0.0562, 0.4753]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Uruguay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6203]\", \"lift_point\": \"[0.0533, 0.6203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1402, 0.6363]\", \"lift_point\": \"[0.1402, 0.6363]\", \"typed_text\": \"\" Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3553, 0.5071]\", \"lift_point\": \"[0.3553, 0.5071]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6605]\", \"lift_point\": \"[0.7839, 0.6605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4753]\", \"lift_point\": \"[0.0562, 0.4753]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Uruguay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6203]\", \"lift_point\": \"[0.0533, 0.6203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1402, 0.6363]\", \"lift_point\": \"[0.1402, 0.6363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3553, 0.5071]\", \"lift_point\": \"[0.3553, 0.5071]\", \"typed_text\": \"\" Goal: What's the news in Uruguay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1327_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0975, 0.4783]\", \"lift_point\": \"[0.0975, 0.4783]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0975, 0.4783]\", \"lift_point\": \"[0.0975, 0.4783]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Spain ?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0975, 0.4783]\", \"lift_point\": \"[0.0975, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Spain?\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0975, 0.4783]\", \"lift_point\": \"[0.0975, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Spain?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0975, 0.4783]\", \"lift_point\": \"[0.0975, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Spain?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6981, 0.9436]\", \"lift_point\": \"[0.6981, 0.9436]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5111, 0.4875]\", \"lift_point\": \"[0.5111, 0.4875]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0975, 0.4783]\", \"lift_point\": \"[0.0975, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Spain?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6981, 0.9436]\", \"lift_point\": \"[0.6981, 0.9436]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1427_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5829, 0.4735]\", \"lift_point\": \"[0.5829, 0.4735]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5829, 0.4735]\", \"lift_point\": \"[0.5829, 0.4735]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3795, 0.6381]\", \"lift_point\": \"[0.3795, 0.6381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5829, 0.4735]\", \"lift_point\": \"[0.5829, 0.4735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3795, 0.6381]\", \"lift_point\": \"[0.3795, 0.6381]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4638, 0.5308]\", \"lift_point\": \"[0.4638, 0.5308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5829, 0.4735]\", \"lift_point\": \"[0.5829, 0.4735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3795, 0.6381]\", \"lift_point\": \"[0.3795, 0.6381]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1113, 0.4897]\", \"lift_point\": \"[0.1113, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5829, 0.4735]\", \"lift_point\": \"[0.5829, 0.4735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3795, 0.6381]\", \"lift_point\": \"[0.3795, 0.6381]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6188, 0.3857]\", \"lift_point\": \"[0.6188, 0.3857]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at O live Garden?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5829, 0.4735]\", \"lift_point\": \"[0.5829, 0.4735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3795, 0.6381]\", \"lift_point\": \"[0.3795, 0.6381]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6188, 0.3857]\", \"lift_point\": \"[0.6188, 0.3857]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1669_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1712_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1712_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4734, 0.5023]\", \"lift_point\": \"[0.4734, 0.5023]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1712_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4734, 0.5023]\", \"lift_point\": \"[0.4734, 0.5023]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1118, 0.5014]\", \"lift_point\": \"[0.1118, 0.5014]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1712_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4734, 0.5023]\", \"lift_point\": \"[0.4734, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1118, 0.5014]\", \"lift_point\": \"[0.1118, 0.5014]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1712_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4734, 0.5023]\", \"lift_point\": \"[0.4734, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1118, 0.5014]\", \"lift_point\": \"[0.1118, 0.5014]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1712_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4734, 0.5023]\", \"lift_point\": \"[0.4734, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1118, 0.5014]\", \"lift_point\": \"[0.1118, 0.5014]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1712_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6125]\", \"lift_point\": \"[0.1388, 0.6125]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6125]\", \"lift_point\": \"[0.1388, 0.6125]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1312, 0.7919]\", \"lift_point\": \"[0.1312, 0.7919]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6125]\", \"lift_point\": \"[0.1388, 0.6125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1312, 0.7919]\", \"lift_point\": \"[0.1312, 0.7919]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.436]\", \"lift_point\": \"[0.8963, 0.436]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.179, 0.3816]\", \"lift_point\": \"[0.179, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6125]\", \"lift_point\": \"[0.1388, 0.6125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1312, 0.7919]\", \"lift_point\": \"[0.1312, 0.7919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.436]\", \"lift_point\": \"[0.8963, 0.436]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.436]\", \"lift_point\": \"[0.8963, 0.436]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.7921]\", \"lift_point\": \"[0.0572, 0.7921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6125]\", \"lift_point\": \"[0.1388, 0.6125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1312, 0.7919]\", \"lift_point\": \"[0.1312, 0.7919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.436]\", \"lift_point\": \"[0.8963, 0.436]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.436]\", \"lift_point\": \"[0.8963, 0.436]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1343, 0.7973]\", \"lift_point\": \"[0.1343, 0.7973]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9391]\", \"lift_point\": \"[0.0569, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"amazon alexa\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6125]\", \"lift_point\": \"[0.1388, 0.6125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1312, 0.7919]\", \"lift_point\": \"[0.1312, 0.7919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.436]\", \"lift_point\": \"[0.8963, 0.436]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.436]\", \"lift_point\": \"[0.8963, 0.436]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1343, 0.7973]\", \"lift_point\": \"[0.1343, 0.7973]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Alexa app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1782_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4401]\", \"lift_point\": \"[0.043, 0.4401]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.6121]\", \"lift_point\": \"[0.749, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4401]\", \"lift_point\": \"[0.043, 0.4401]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4401]\", \"lift_point\": \"[0.043, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4401]\", \"lift_point\": \"[0.043, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1287, 0.5866]\", \"lift_point\": \"[0.1287, 0.5866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4401]\", \"lift_point\": \"[0.043, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Barbados?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1287, 0.5866]\", \"lift_point\": \"[0.1287, 0.5866]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1837_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5083, 0.1503]\", \"lift_point\": \"[0.5083, 0.1503]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5083, 0.1503]\", \"lift_point\": \"[0.5083, 0.1503]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0641]\", \"lift_point\": \"[0.0703, 0.0641]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5083, 0.1503]\", \"lift_point\": \"[0.5083, 0.1503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0641]\", \"lift_point\": \"[0.0703, 0.0641]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.5141]\", \"lift_point\": \"[0.8975, 0.5141]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5083, 0.1503]\", \"lift_point\": \"[0.5083, 0.1503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0641]\", \"lift_point\": \"[0.0703, 0.0641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.5141]\", \"lift_point\": \"[0.8975, 0.5141]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3199, 0.6084]\", \"lift_point\": \"[0.3199, 0.6084]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5083, 0.1503]\", \"lift_point\": \"[0.5083, 0.1503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0641]\", \"lift_point\": \"[0.0703, 0.0641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.5141]\", \"lift_point\": \"[0.8975, 0.5141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3199, 0.6084]\", \"lift_point\": \"[0.3199, 0.6084]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.5209]\", \"lift_point\": \"[0.4096, 0.5209]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5083, 0.1503]\", \"lift_point\": \"[0.5083, 0.1503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0641]\", \"lift_point\": \"[0.0703, 0.0641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.5141]\", \"lift_point\": \"[0.8975, 0.5141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3199, 0.6084]\", \"lift_point\": \"[0.3199, 0.6084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.5209]\", \"lift_point\": \"[0.4096, 0.5209]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6795, 0.7597]\", \"lift_point\": \"[0.6795, 0.7597]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5083, 0.1503]\", \"lift_point\": \"[0.5083, 0.1503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0641]\", \"lift_point\": \"[0.0703, 0.0641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.5141]\", \"lift_point\": \"[0.8975, 0.5141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3199, 0.6084]\", \"lift_point\": \"[0.3199, 0.6084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.5209]\", \"lift_point\": \"[0.4096, 0.5209]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6795, 0.7597]\", \"lift_point\": \"[0.6795, 0.7597]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1984_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new tab in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7873, 0.6244]\", \"lift_point\": \"[0.7873, 0.6244]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1988_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7873, 0.6244]\", \"lift_point\": \"[0.7873, 0.6244]\", \"typed_text\": \"\" Goal: Open a new tab in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8212]\", \"lift_point\": \"[0.0533, 0.8212]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1988_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7873, 0.6244]\", \"lift_point\": \"[0.7873, 0.6244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8212]\", \"lift_point\": \"[0.0533, 0.8212]\", \"typed_text\": \"\" Goal: Open a new tab in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.08]\", \"lift_point\": \"[0.0669, 0.08]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1988_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7873, 0.6244]\", \"lift_point\": \"[0.7873, 0.6244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8212]\", \"lift_point\": \"[0.0533, 0.8212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.08]\", \"lift_point\": \"[0.0669, 0.08]\", \"typed_text\": \"\" Goal: Open a new tab in the chrome app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/1988_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.6866]\", \"lift_point\": \"[0.5058, 0.6866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.6866]\", \"lift_point\": \"[0.5058, 0.6866]\", \"typed_text\": \"\" Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6236]\", \"lift_point\": \"[0.1233, 0.6236]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.6866]\", \"lift_point\": \"[0.5058, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6236]\", \"lift_point\": \"[0.1233, 0.6236]\", \"typed_text\": \"\" Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.9196]\", \"lift_point\": \"[0.0716, 0.9196]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.6866]\", \"lift_point\": \"[0.5058, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6236]\", \"lift_point\": \"[0.1233, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.9196]\", \"lift_point\": \"[0.0716, 0.9196]\", \"typed_text\": \"\" Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Sony TV?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.6866]\", \"lift_point\": \"[0.5058, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6236]\", \"lift_point\": \"[0.1233, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.9196]\", \"lift_point\": \"[0.0716, 0.9196]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Sony TV?\" Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1222, 0.3466]\", \"lift_point\": \"[0.1222, 0.3466]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.6866]\", \"lift_point\": \"[0.5058, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6236]\", \"lift_point\": \"[0.1233, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.9196]\", \"lift_point\": \"[0.0716, 0.9196]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Sony TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1222, 0.3466]\", \"lift_point\": \"[0.1222, 0.3466]\", \"typed_text\": \"\" Goal: What's the price of the Sony TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2035_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9052, 0.8727]\", \"lift_point\": \"[0.9052, 0.8727]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9052, 0.8727]\", \"lift_point\": \"[0.9052, 0.8727]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9052, 0.8727]\", \"lift_point\": \"[0.9052, 0.8727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2241, 0.2157]\", \"lift_point\": \"[0.2241, 0.2157]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9052, 0.8727]\", \"lift_point\": \"[0.9052, 0.8727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2241, 0.2157]\", \"lift_point\": \"[0.2241, 0.2157]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.9342]\", \"lift_point\": \"[0.0624, 0.9342]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9052, 0.8727]\", \"lift_point\": \"[0.9052, 0.8727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2241, 0.2157]\", \"lift_point\": \"[0.2241, 0.2157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.9342]\", \"lift_point\": \"[0.0624, 0.9342]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.165, 0.7366]\", \"lift_point\": \"[0.165, 0.7366]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.369, 0.4993]\", \"lift_point\": \"[0.369, 0.4993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8796, 0.5176]\", \"lift_point\": \"[0.8796, 0.5176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9052, 0.8727]\", \"lift_point\": \"[0.9052, 0.8727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2241, 0.2157]\", \"lift_point\": \"[0.2241, 0.2157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.9342]\", \"lift_point\": \"[0.0624, 0.9342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.165, 0.7366]\", \"lift_point\": \"[0.165, 0.7366]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2057_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2100_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2100_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.8874]\", \"lift_point\": \"[0.7839, 0.8874]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2100_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.8874]\", \"lift_point\": \"[0.7839, 0.8874]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2100_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4934, 0.705]\", \"lift_point\": \"[0.4934, 0.705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4934, 0.705]\", \"lift_point\": \"[0.4934, 0.705]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.6287]\", \"lift_point\": \"[0.1292, 0.6287]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4934, 0.705]\", \"lift_point\": \"[0.4934, 0.705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.6287]\", \"lift_point\": \"[0.1292, 0.6287]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.9183]\", \"lift_point\": \"[0.0555, 0.9183]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4934, 0.705]\", \"lift_point\": \"[0.4934, 0.705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.6287]\", \"lift_point\": \"[0.1292, 0.6287]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.9183]\", \"lift_point\": \"[0.0555, 0.9183]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronics store\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4934, 0.705]\", \"lift_point\": \"[0.4934, 0.705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.6287]\", \"lift_point\": \"[0.1292, 0.6287]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.9183]\", \"lift_point\": \"[0.0555, 0.9183]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronics store\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.544]\", \"lift_point\": \"[0.1161, 0.544]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4934, 0.705]\", \"lift_point\": \"[0.4934, 0.705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.6287]\", \"lift_point\": \"[0.1292, 0.6287]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.9183]\", \"lift_point\": \"[0.0555, 0.9183]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronics store\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.544]\", \"lift_point\": \"[0.1161, 0.544]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2115_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.797]\", \"lift_point\": \"[0.1234, 0.797]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.797]\", \"lift_point\": \"[0.1234, 0.797]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9237]\", \"lift_point\": \"[0.0724, 0.9237]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.797]\", \"lift_point\": \"[0.1234, 0.797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9237]\", \"lift_point\": \"[0.0724, 0.9237]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Home Depot?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.797]\", \"lift_point\": \"[0.1234, 0.797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9237]\", \"lift_point\": \"[0.0724, 0.9237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Home Depot?\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1761, 0.5539]\", \"lift_point\": \"[0.1761, 0.5539]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.797]\", \"lift_point\": \"[0.1234, 0.797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9237]\", \"lift_point\": \"[0.0724, 0.9237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Home Depot?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1761, 0.5539]\", \"lift_point\": \"[0.1761, 0.5539]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.5864]\", \"lift_point\": \"[0.6475, 0.5864]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.797]\", \"lift_point\": \"[0.1234, 0.797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9237]\", \"lift_point\": \"[0.0724, 0.9237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Home Depot?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1761, 0.5539]\", \"lift_point\": \"[0.1761, 0.5539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.5864]\", \"lift_point\": \"[0.6475, 0.5864]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1838, 0.2588]\", \"lift_point\": \"[0.1838, 0.2588]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.711]\", \"lift_point\": \"[0.4937, 0.711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.797]\", \"lift_point\": \"[0.1234, 0.797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9237]\", \"lift_point\": \"[0.0724, 0.9237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Home Depot?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1761, 0.5539]\", \"lift_point\": \"[0.1761, 0.5539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.5864]\", \"lift_point\": \"[0.6475, 0.5864]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1838, 0.2588]\", \"lift_point\": \"[0.1838, 0.2588]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2163_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the files app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2397_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the files app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2397_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the files app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.4789]\", \"lift_point\": \"[0.4698, 0.4789]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2397_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.4789]\", \"lift_point\": \"[0.4698, 0.4789]\", \"typed_text\": \"\" Goal: Open the files app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2397_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4741, 0.854]\", \"lift_point\": \"[0.4741, 0.854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4741, 0.854]\", \"lift_point\": \"[0.4741, 0.854]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.7939]\", \"lift_point\": \"[0.1283, 0.7939]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4741, 0.854]\", \"lift_point\": \"[0.4741, 0.854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.7939]\", \"lift_point\": \"[0.1283, 0.7939]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9105]\", \"lift_point\": \"[0.0549, 0.9105]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4741, 0.854]\", \"lift_point\": \"[0.4741, 0.854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.7939]\", \"lift_point\": \"[0.1283, 0.7939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9105]\", \"lift_point\": \"[0.0549, 0.9105]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rateagainst the Swiss Franc?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4741, 0.854]\", \"lift_point\": \"[0.4741, 0.854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.7939]\", \"lift_point\": \"[0.1283, 0.7939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9105]\", \"lift_point\": \"[0.0549, 0.9105]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rateagainst the Swiss Franc?\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1165, 0.7526]\", \"lift_point\": \"[0.1165, 0.7526]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4741, 0.854]\", \"lift_point\": \"[0.4741, 0.854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.7939]\", \"lift_point\": \"[0.1283, 0.7939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9105]\", \"lift_point\": \"[0.0549, 0.9105]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rateagainst the Swiss Franc?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1165, 0.7526]\", \"lift_point\": \"[0.1165, 0.7526]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2466_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this evening?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2480_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this evening?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4921, 0.8521]\", \"lift_point\": \"[0.4921, 0.8521]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2480_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4921, 0.8521]\", \"lift_point\": \"[0.4921, 0.8521]\", \"typed_text\": \"\" Goal: What's the news this evening?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1316, 0.703]\", \"lift_point\": \"[0.1316, 0.703]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2480_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4921, 0.8521]\", \"lift_point\": \"[0.4921, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1316, 0.703]\", \"lift_point\": \"[0.1316, 0.703]\", \"typed_text\": \"\" Goal: What's the news this evening?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.8909]\", \"lift_point\": \"[0.0721, 0.8909]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2480_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4921, 0.8521]\", \"lift_point\": \"[0.4921, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1316, 0.703]\", \"lift_point\": \"[0.1316, 0.703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.8909]\", \"lift_point\": \"[0.0721, 0.8909]\", \"typed_text\": \"\" Goal: What's the news this evening?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this evening\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2480_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4921, 0.8521]\", \"lift_point\": \"[0.4921, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1316, 0.703]\", \"lift_point\": \"[0.1316, 0.703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.8909]\", \"lift_point\": \"[0.0721, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this evening\" Goal: What's the news this evening?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2493, 0.5909]\", \"lift_point\": \"[0.2493, 0.5909]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2480_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4921, 0.8521]\", \"lift_point\": \"[0.4921, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1316, 0.703]\", \"lift_point\": \"[0.1316, 0.703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.8909]\", \"lift_point\": \"[0.0721, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this evening\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2493, 0.5909]\", \"lift_point\": \"[0.2493, 0.5909]\", \"typed_text\": \"\" Goal: What's the news this evening?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2480_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4646, 0.8911]\", \"lift_point\": \"[0.4646, 0.8911]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4646, 0.8911]\", \"lift_point\": \"[0.4646, 0.8911]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.7459]\", \"lift_point\": \"[0.1309, 0.7459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4646, 0.8911]\", \"lift_point\": \"[0.4646, 0.8911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.7459]\", \"lift_point\": \"[0.1309, 0.7459]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.9202]\", \"lift_point\": \"[0.0642, 0.9202]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4646, 0.8911]\", \"lift_point\": \"[0.4646, 0.8911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.7459]\", \"lift_point\": \"[0.1309, 0.7459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.9202]\", \"lift_point\": \"[0.0642, 0.9202]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Philadelphia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4646, 0.8911]\", \"lift_point\": \"[0.4646, 0.8911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.7459]\", \"lift_point\": \"[0.1309, 0.7459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.9202]\", \"lift_point\": \"[0.0642, 0.9202]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Philadelphia\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1145, 0.5032]\", \"lift_point\": \"[0.1145, 0.5032]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4646, 0.8911]\", \"lift_point\": \"[0.4646, 0.8911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.7459]\", \"lift_point\": \"[0.1309, 0.7459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.9202]\", \"lift_point\": \"[0.0642, 0.9202]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Philadelphia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1145, 0.5032]\", \"lift_point\": \"[0.1145, 0.5032]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1958, 0.8918]\", \"lift_point\": \"[0.1958, 0.8918]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4646, 0.8911]\", \"lift_point\": \"[0.4646, 0.8911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.7459]\", \"lift_point\": \"[0.1309, 0.7459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.9202]\", \"lift_point\": \"[0.0642, 0.9202]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Philadelphia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1145, 0.5032]\", \"lift_point\": \"[0.1145, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1958, 0.8918]\", \"lift_point\": \"[0.1958, 0.8918]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2543_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2549_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.7157]\", \"lift_point\": \"[0.7561, 0.7157]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2549_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.7157]\", \"lift_point\": \"[0.7561, 0.7157]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.8402]\", \"lift_point\": \"[0.0622, 0.8402]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2549_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.7157]\", \"lift_point\": \"[0.7561, 0.7157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.8402]\", \"lift_point\": \"[0.0622, 0.8402]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Katy Perry video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2549_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.7157]\", \"lift_point\": \"[0.7561, 0.7157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.8402]\", \"lift_point\": \"[0.0622, 0.8402]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Katy Perry video\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0884, 0.584]\", \"lift_point\": \"[0.0884, 0.584]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2549_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.7157]\", \"lift_point\": \"[0.7561, 0.7157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.8402]\", \"lift_point\": \"[0.0622, 0.8402]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Katy Perry video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0884, 0.584]\", \"lift_point\": \"[0.0884, 0.584]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2549_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6932]\", \"lift_point\": \"[0.7923, 0.6932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6932]\", \"lift_point\": \"[0.7923, 0.6932]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7972, 0.6824]\", \"lift_point\": \"[0.7972, 0.6824]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6932]\", \"lift_point\": \"[0.7923, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7972, 0.6824]\", \"lift_point\": \"[0.7972, 0.6824]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6824]\", \"lift_point\": \"[0.7985, 0.6824]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6932]\", \"lift_point\": \"[0.7923, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7972, 0.6824]\", \"lift_point\": \"[0.7972, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6824]\", \"lift_point\": \"[0.7985, 0.6824]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.6006]\", \"lift_point\": \"[0.0615, 0.6006]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6932]\", \"lift_point\": \"[0.7923, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7972, 0.6824]\", \"lift_point\": \"[0.7972, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6824]\", \"lift_point\": \"[0.7985, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.6006]\", \"lift_point\": \"[0.0615, 0.6006]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for my phone\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6932]\", \"lift_point\": \"[0.7923, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7972, 0.6824]\", \"lift_point\": \"[0.7972, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6824]\", \"lift_point\": \"[0.7985, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.6006]\", \"lift_point\": \"[0.0615, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for my phone\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.3267]\", \"lift_point\": \"[0.1436, 0.3267]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.786, 0.6713]\", \"lift_point\": \"[0.786, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.6932]\", \"lift_point\": \"[0.7923, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7972, 0.6824]\", \"lift_point\": \"[0.7972, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6824]\", \"lift_point\": \"[0.7985, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.6006]\", \"lift_point\": \"[0.0615, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for my phone\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.3267]\", \"lift_point\": \"[0.1436, 0.3267]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2641_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8012, 0.6957]\", \"lift_point\": \"[0.8012, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7813, 0.6517]\", \"lift_point\": \"[0.7813, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.6339]\", \"lift_point\": \"[0.0642, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9935, 0.466]\", \"lift_point\": \"[0.9935, 0.466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4465, 0.9944]\", \"lift_point\": \"[0.4465, 0.9944]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.2568]\", \"lift_point\": \"[0.0642, 0.2568]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.3084]\", \"lift_point\": \"[0.0622, 0.3084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.2568]\", \"lift_point\": \"[0.0642, 0.2568]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.2568]\", \"lift_point\": \"[0.0642, 0.2568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.5584]\", \"lift_point\": \"[0.1434, 0.5584]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.1702]\", \"lift_point\": \"[0.0509, 0.1702]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.2568]\", \"lift_point\": \"[0.0642, 0.2568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.5584]\", \"lift_point\": \"[0.1434, 0.5584]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.2568]\", \"lift_point\": \"[0.0642, 0.2568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.5584]\", \"lift_point\": \"[0.1434, 0.5584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.2568]\", \"lift_point\": \"[0.0642, 0.2568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.5584]\", \"lift_point\": \"[0.1434, 0.5584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4244, 0.5674]\", \"lift_point\": \"[0.4244, 0.5674]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.6879]\", \"lift_point\": \"[0.7909, 0.6879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0642, 0.2568]\", \"lift_point\": \"[0.0642, 0.2568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.5584]\", \"lift_point\": \"[0.1434, 0.5584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4244, 0.5674]\", \"lift_point\": \"[0.4244, 0.5674]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2733_22.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5494]\", \"lift_point\": \"[0.0597, 0.5494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5494]\", \"lift_point\": \"[0.0597, 0.5494]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather like in New York\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5494]\", \"lift_point\": \"[0.0597, 0.5494]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather like in New York\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1428, 0.4671]\", \"lift_point\": \"[0.1428, 0.4671]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7879, 0.6835]\", \"lift_point\": \"[0.7879, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6835]\", \"lift_point\": \"[0.7891, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5494]\", \"lift_point\": \"[0.0597, 0.5494]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather like in New York\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1428, 0.4671]\", \"lift_point\": \"[0.1428, 0.4671]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2742_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5808]\", \"lift_point\": \"[0.0555, 0.5808]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5808]\", \"lift_point\": \"[0.0555, 0.5808]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.8933]\", \"lift_point\": \"[0.0647, 0.8933]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5808]\", \"lift_point\": \"[0.0555, 0.5808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.8933]\", \"lift_point\": \"[0.0647, 0.8933]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest IKEA\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5808]\", \"lift_point\": \"[0.0555, 0.5808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.8933]\", \"lift_point\": \"[0.0647, 0.8933]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest IKEA\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.6285]\", \"lift_point\": \"[0.1419, 0.6285]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6787]\", \"lift_point\": \"[0.7898, 0.6787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5808]\", \"lift_point\": \"[0.0555, 0.5808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.8933]\", \"lift_point\": \"[0.0647, 0.8933]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest IKEA\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.6285]\", \"lift_point\": \"[0.1419, 0.6285]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1382, 0.555]\", \"lift_point\": \"[0.1382, 0.555]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7937, 0.6907]\", \"lift_point\": \"[0.7937, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7962, 0.6827]\", \"lift_point\": \"[0.7962, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5808]\", \"lift_point\": \"[0.0555, 0.5808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.8933]\", \"lift_point\": \"[0.0647, 0.8933]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest IKEA\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.6285]\", \"lift_point\": \"[0.1419, 0.6285]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1382, 0.555]\", \"lift_point\": \"[0.1382, 0.555]\", \"typed_text\": \"\" Goal: How do I get to the nearest IKEA?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2874_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.5173]\", \"lift_point\": \"[0.1396, 0.5173]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.5173]\", \"lift_point\": \"[0.1396, 0.5173]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6844]\", \"lift_point\": \"[0.7996, 0.6844]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.5173]\", \"lift_point\": \"[0.1396, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6932, 0.5376]\", \"lift_point\": \"[0.6932, 0.5376]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.7008]\", \"lift_point\": \"[0.7907, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.5173]\", \"lift_point\": \"[0.1396, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6932, 0.5376]\", \"lift_point\": \"[0.6932, 0.5376]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6875]\", \"lift_point\": \"[0.7874, 0.6875]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.5173]\", \"lift_point\": \"[0.1396, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6932, 0.5376]\", \"lift_point\": \"[0.6932, 0.5376]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7013, 0.4971]\", \"lift_point\": \"[0.7013, 0.4971]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.5807]\", \"lift_point\": \"[0.0646, 0.5807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.5173]\", \"lift_point\": \"[0.1396, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6932, 0.5376]\", \"lift_point\": \"[0.6932, 0.5376]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7013, 0.4971]\", \"lift_point\": \"[0.7013, 0.4971]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7036, 0.4831]\", \"lift_point\": \"[0.7036, 0.4831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9159]\", \"lift_point\": \"[0.066, 0.9159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.5173]\", \"lift_point\": \"[0.1396, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6932, 0.5376]\", \"lift_point\": \"[0.6932, 0.5376]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7013, 0.4971]\", \"lift_point\": \"[0.7013, 0.4971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7036, 0.4831]\", \"lift_point\": \"[0.7036, 0.4831]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2930_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5317]\", \"lift_point\": \"[0.0665, 0.5317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5317]\", \"lift_point\": \"[0.0665, 0.5317]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ESPN app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5317]\", \"lift_point\": \"[0.0665, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ESPN app\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1459, 0.2763]\", \"lift_point\": \"[0.1459, 0.2763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5317]\", \"lift_point\": \"[0.0665, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ESPN app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1459, 0.2763]\", \"lift_point\": \"[0.1459, 0.2763]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8292]\", \"lift_point\": \"[0.1595, 0.8292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5317]\", \"lift_point\": \"[0.0665, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ESPN app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1459, 0.2763]\", \"lift_point\": \"[0.1459, 0.2763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8292]\", \"lift_point\": \"[0.1595, 0.8292]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8953, 0.5424]\", \"lift_point\": \"[0.8953, 0.5424]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8761]\", \"lift_point\": \"[0.6348, 0.8761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5317]\", \"lift_point\": \"[0.0665, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ESPN app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1459, 0.2763]\", \"lift_point\": \"[0.1459, 0.2763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8292]\", \"lift_point\": \"[0.1595, 0.8292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8953, 0.5424]\", \"lift_point\": \"[0.8953, 0.5424]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9034, 0.3615]\", \"lift_point\": \"[0.9034, 0.3615]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8734]\", \"lift_point\": \"[0.6423, 0.8734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6472, 0.8598]\", \"lift_point\": \"[0.6472, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5317]\", \"lift_point\": \"[0.0665, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ESPN app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1459, 0.2763]\", \"lift_point\": \"[0.1459, 0.2763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8292]\", \"lift_point\": \"[0.1595, 0.8292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8953, 0.5424]\", \"lift_point\": \"[0.8953, 0.5424]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9034, 0.3615]\", \"lift_point\": \"[0.9034, 0.3615]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/2971_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0779, 0.6367]\", \"lift_point\": \"[0.0779, 0.6367]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0779, 0.6367]\", \"lift_point\": \"[0.0779, 0.6367]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0779, 0.6367]\", \"lift_point\": \"[0.0779, 0.6367]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.533]\", \"lift_point\": \"[0.134, 0.533]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6728]\", \"lift_point\": \"[0.7998, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0779, 0.6367]\", \"lift_point\": \"[0.0779, 0.6367]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.533]\", \"lift_point\": \"[0.134, 0.533]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6692]\", \"lift_point\": \"[0.7859, 0.6692]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0779, 0.6367]\", \"lift_point\": \"[0.0779, 0.6367]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.533]\", \"lift_point\": \"[0.134, 0.533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7935, 0.6859]\", \"lift_point\": \"[0.7935, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0779, 0.6367]\", \"lift_point\": \"[0.0779, 0.6367]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.533]\", \"lift_point\": \"[0.134, 0.533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2044, 0.184]\", \"lift_point\": \"[0.2044, 0.184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6859]\", \"lift_point\": \"[0.7911, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6859]\", \"lift_point\": \"[0.7898, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0779, 0.6367]\", \"lift_point\": \"[0.0779, 0.6367]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.533]\", \"lift_point\": \"[0.134, 0.533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2044, 0.184]\", \"lift_point\": \"[0.2044, 0.184]\", \"typed_text\": \"\" Goal: What's the news about the US?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3028_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.1438]\", \"lift_point\": \"[0.6505, 0.1438]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.1438]\", \"lift_point\": \"[0.6505, 0.1438]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8877]\", \"lift_point\": \"[0.0638, 0.8877]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.1438]\", \"lift_point\": \"[0.6505, 0.1438]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8877]\", \"lift_point\": \"[0.0638, 0.8877]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.3859]\", \"lift_point\": \"[0.0566, 0.3859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3998, 0.9869]\", \"lift_point\": \"[0.3998, 0.9869]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.1438]\", \"lift_point\": \"[0.6505, 0.1438]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8877]\", \"lift_point\": \"[0.0638, 0.8877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.3859]\", \"lift_point\": \"[0.0566, 0.3859]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"restaurant in Dallas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.1438]\", \"lift_point\": \"[0.6505, 0.1438]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8877]\", \"lift_point\": \"[0.0638, 0.8877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.3859]\", \"lift_point\": \"[0.0566, 0.3859]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"restaurant in Dallas\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1539, 0.4027]\", \"lift_point\": \"[0.1539, 0.4027]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6376, 0.1327]\", \"lift_point\": \"[0.6376, 0.1327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.1438]\", \"lift_point\": \"[0.6505, 0.1438]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8877]\", \"lift_point\": \"[0.0638, 0.8877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.3859]\", \"lift_point\": \"[0.0566, 0.3859]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"restaurant in Dallas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1539, 0.4027]\", \"lift_point\": \"[0.1539, 0.4027]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3064_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.6731]\", \"lift_point\": \"[0.0663, 0.6731]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.6731]\", \"lift_point\": \"[0.0663, 0.6731]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9356]\", \"lift_point\": \"[0.0672, 0.9356]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.6731]\", \"lift_point\": \"[0.0663, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9356]\", \"lift_point\": \"[0.0672, 0.9356]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.6731]\", \"lift_point\": \"[0.0663, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9356]\", \"lift_point\": \"[0.0672, 0.9356]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1377, 0.2721]\", \"lift_point\": \"[0.1377, 0.2721]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.6731]\", \"lift_point\": \"[0.0663, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9356]\", \"lift_point\": \"[0.0672, 0.9356]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1377, 0.2721]\", \"lift_point\": \"[0.1377, 0.2721]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.7041]\", \"lift_point\": \"[0.7912, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.6731]\", \"lift_point\": \"[0.0663, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9356]\", \"lift_point\": \"[0.0672, 0.9356]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1377, 0.2721]\", \"lift_point\": \"[0.1377, 0.2721]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.5026]\", \"lift_point\": \"[0.4338, 0.5026]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6874]\", \"lift_point\": \"[0.7849, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.6794]\", \"lift_point\": \"[0.7788, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.6731]\", \"lift_point\": \"[0.0663, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9356]\", \"lift_point\": \"[0.0672, 0.9356]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1377, 0.2721]\", \"lift_point\": \"[0.1377, 0.2721]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.5026]\", \"lift_point\": \"[0.4338, 0.5026]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3192_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.8989]\", \"lift_point\": \"[0.0598, 0.8989]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.8989]\", \"lift_point\": \"[0.0598, 0.8989]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Austin\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.8989]\", \"lift_point\": \"[0.0598, 0.8989]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Austin\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6327]\", \"lift_point\": \"[0.1388, 0.6327]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.8989]\", \"lift_point\": \"[0.0598, 0.8989]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Austin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6327]\", \"lift_point\": \"[0.1388, 0.6327]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.8989]\", \"lift_point\": \"[0.0598, 0.8989]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Austin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6327]\", \"lift_point\": \"[0.1388, 0.6327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.8989]\", \"lift_point\": \"[0.0598, 0.8989]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Austin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6327]\", \"lift_point\": \"[0.1388, 0.6327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5339, 0.5024]\", \"lift_point\": \"[0.5339, 0.5024]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7874, 0.6476]\", \"lift_point\": \"[0.7874, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4847]\", \"lift_point\": \"[0.0578, 0.4847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.8989]\", \"lift_point\": \"[0.0598, 0.8989]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Austin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.6327]\", \"lift_point\": \"[0.1388, 0.6327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5339, 0.5024]\", \"lift_point\": \"[0.5339, 0.5024]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3394_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of the United Kingdom\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of the United Kingdom\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.9913]\", \"lift_point\": \"[0.3263, 0.9913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of the United Kingdom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.9913]\", \"lift_point\": \"[0.3263, 0.9913]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4773]\", \"lift_point\": \"[0.1363, 0.4773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7867, 0.6739]\", \"lift_point\": \"[0.7867, 0.6739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of the United Kingdom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.9913]\", \"lift_point\": \"[0.3263, 0.9913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4773]\", \"lift_point\": \"[0.1363, 0.4773]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of the United Kingdom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.9913]\", \"lift_point\": \"[0.3263, 0.9913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4773]\", \"lift_point\": \"[0.1363, 0.4773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.4577]\", \"lift_point\": \"[0.5958, 0.4577]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6932]\", \"lift_point\": \"[0.7854, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5945]\", \"lift_point\": \"[0.0665, 0.5945]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of the United Kingdom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.9913]\", \"lift_point\": \"[0.3263, 0.9913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4773]\", \"lift_point\": \"[0.1363, 0.4773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.4577]\", \"lift_point\": \"[0.5958, 0.4577]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3444_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2121, 0.6359]\", \"lift_point\": \"[0.2121, 0.6359]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2121, 0.6359]\", \"lift_point\": \"[0.2121, 0.6359]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.148, 0.9353]\", \"lift_point\": \"[0.148, 0.9353]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2121, 0.6359]\", \"lift_point\": \"[0.2121, 0.6359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.148, 0.9353]\", \"lift_point\": \"[0.148, 0.9353]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Best Buy?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2121, 0.6359]\", \"lift_point\": \"[0.2121, 0.6359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.148, 0.9353]\", \"lift_point\": \"[0.148, 0.9353]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Best Buy?\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.6859]\", \"lift_point\": \"[0.7971, 0.6859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7046]\", \"lift_point\": \"[0.7974, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2121, 0.6359]\", \"lift_point\": \"[0.2121, 0.6359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.148, 0.9353]\", \"lift_point\": \"[0.148, 0.9353]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Best Buy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/3930_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4007_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.672]\", \"lift_point\": \"[0.8027, 0.672]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4007_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.672]\", \"lift_point\": \"[0.8027, 0.672]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2187, 0.6753]\", \"lift_point\": \"[0.2187, 0.6753]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4007_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.672]\", \"lift_point\": \"[0.8027, 0.672]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2187, 0.6753]\", \"lift_point\": \"[0.2187, 0.6753]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1355, 0.9371]\", \"lift_point\": \"[0.1355, 0.9371]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4007_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.672]\", \"lift_point\": \"[0.8027, 0.672]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2187, 0.6753]\", \"lift_point\": \"[0.2187, 0.6753]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1355, 0.9371]\", \"lift_point\": \"[0.1355, 0.9371]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Diego?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4007_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.672]\", \"lift_point\": \"[0.8027, 0.672]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2187, 0.6753]\", \"lift_point\": \"[0.2187, 0.6753]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1355, 0.9371]\", \"lift_point\": \"[0.1355, 0.9371]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Diego?\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4007_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.672]\", \"lift_point\": \"[0.8027, 0.672]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2187, 0.6753]\", \"lift_point\": \"[0.2187, 0.6753]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1355, 0.9371]\", \"lift_point\": \"[0.1355, 0.9371]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Diego?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4007_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this month?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.7676]\", \"lift_point\": \"[0.7893, 0.7676]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.7676]\", \"lift_point\": \"[0.7893, 0.7676]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2045, 0.7962]\", \"lift_point\": \"[0.2045, 0.7962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3444, 0.5381]\", \"lift_point\": \"[0.3444, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.7676]\", \"lift_point\": \"[0.7893, 0.7676]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2045, 0.7962]\", \"lift_point\": \"[0.2045, 0.7962]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4103_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4274_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4274_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8224, 0.5986]\", \"lift_point\": \"[0.8224, 0.5986]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4274_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8224, 0.5986]\", \"lift_point\": \"[0.8224, 0.5986]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4008]\", \"lift_point\": \"[0.0518, 0.4008]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4274_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8224, 0.5986]\", \"lift_point\": \"[0.8224, 0.5986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4008]\", \"lift_point\": \"[0.0518, 0.4008]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in delhi\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4274_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8224, 0.5986]\", \"lift_point\": \"[0.8224, 0.5986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4008]\", \"lift_point\": \"[0.0518, 0.4008]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in delhi\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1265, 0.1416]\", \"lift_point\": \"[0.1265, 0.1416]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4274_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8224, 0.5986]\", \"lift_point\": \"[0.8224, 0.5986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4008]\", \"lift_point\": \"[0.0518, 0.4008]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in delhi\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1265, 0.1416]\", \"lift_point\": \"[0.1265, 0.1416]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4274_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.5924]\", \"lift_point\": \"[0.8276, 0.5924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4324_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.5924]\", \"lift_point\": \"[0.8276, 0.5924]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1873, 0.3683]\", \"lift_point\": \"[0.1873, 0.3683]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4324_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.5924]\", \"lift_point\": \"[0.8276, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1873, 0.3683]\", \"lift_point\": \"[0.1873, 0.3683]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.9421]\", \"lift_point\": \"[0.121, 0.9421]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4324_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.5924]\", \"lift_point\": \"[0.8276, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1873, 0.3683]\", \"lift_point\": \"[0.1873, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.9421]\", \"lift_point\": \"[0.121, 0.9421]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Brazil\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4324_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.5924]\", \"lift_point\": \"[0.8276, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1873, 0.3683]\", \"lift_point\": \"[0.1873, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.9421]\", \"lift_point\": \"[0.121, 0.9421]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Brazil\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1693, 0.3372]\", \"lift_point\": \"[0.1693, 0.3372]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4324_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.5924]\", \"lift_point\": \"[0.8276, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1873, 0.3683]\", \"lift_point\": \"[0.1873, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.9421]\", \"lift_point\": \"[0.121, 0.9421]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Brazil\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1693, 0.3372]\", \"lift_point\": \"[0.1693, 0.3372]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5779, 0.1509]\", \"lift_point\": \"[0.5779, 0.1509]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4324_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.5924]\", \"lift_point\": \"[0.8276, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1873, 0.3683]\", \"lift_point\": \"[0.1873, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.9421]\", \"lift_point\": \"[0.121, 0.9421]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Brazil\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1693, 0.3372]\", \"lift_point\": \"[0.1693, 0.3372]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5779, 0.1509]\", \"lift_point\": \"[0.5779, 0.1509]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4324_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.8781]\", \"lift_point\": \"[0.1779, 0.8781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.8781]\", \"lift_point\": \"[0.1779, 0.8781]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.279]\", \"lift_point\": \"[0.1161, 0.279]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.8781]\", \"lift_point\": \"[0.1779, 0.8781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.279]\", \"lift_point\": \"[0.1161, 0.279]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Italy\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.8781]\", \"lift_point\": \"[0.1779, 0.8781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.279]\", \"lift_point\": \"[0.1161, 0.279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Italy\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1834, 0.2831]\", \"lift_point\": \"[0.1834, 0.2831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.8781]\", \"lift_point\": \"[0.1779, 0.8781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.279]\", \"lift_point\": \"[0.1161, 0.279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Italy\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1834, 0.2831]\", \"lift_point\": \"[0.1834, 0.2831]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2408, 0.0333]\", \"lift_point\": \"[0.2408, 0.0333]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.8781]\", \"lift_point\": \"[0.1779, 0.8781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.279]\", \"lift_point\": \"[0.1161, 0.279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Italy\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1834, 0.2831]\", \"lift_point\": \"[0.1834, 0.2831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2408, 0.0333]\", \"lift_point\": \"[0.2408, 0.0333]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4306, 0.2395]\", \"lift_point\": \"[0.4306, 0.2395]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.613]\", \"lift_point\": \"[0.8322, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1779, 0.8781]\", \"lift_point\": \"[0.1779, 0.8781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.279]\", \"lift_point\": \"[0.1161, 0.279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Italy\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1834, 0.2831]\", \"lift_point\": \"[0.1834, 0.2831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2408, 0.0333]\", \"lift_point\": \"[0.2408, 0.0333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4306, 0.2395]\", \"lift_point\": \"[0.4306, 0.2395]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4496_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4586_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6468, 0.3841]\", \"lift_point\": \"[0.6468, 0.3841]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4586_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6468, 0.3841]\", \"lift_point\": \"[0.6468, 0.3841]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5954, 0.5203]\", \"lift_point\": \"[0.5954, 0.5203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4586_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6468, 0.3841]\", \"lift_point\": \"[0.6468, 0.3841]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5954, 0.5203]\", \"lift_point\": \"[0.5954, 0.5203]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4586_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6468, 0.3841]\", \"lift_point\": \"[0.6468, 0.3841]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5954, 0.5203]\", \"lift_point\": \"[0.5954, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4586_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6468, 0.3841]\", \"lift_point\": \"[0.6468, 0.3841]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5954, 0.5203]\", \"lift_point\": \"[0.5954, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8591, 0.8864]\", \"lift_point\": \"[0.8591, 0.8864]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4586_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6468, 0.3841]\", \"lift_point\": \"[0.6468, 0.3841]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5954, 0.5203]\", \"lift_point\": \"[0.5954, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9472, 0.4803]\", \"lift_point\": \"[0.9472, 0.4803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8591, 0.8864]\", \"lift_point\": \"[0.8591, 0.8864]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4586_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4727_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8212, 0.6184]\", \"lift_point\": \"[0.8212, 0.6184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4727_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8212, 0.6184]\", \"lift_point\": \"[0.8212, 0.6184]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9288]\", \"lift_point\": \"[0.058, 0.9288]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4727_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8212, 0.6184]\", \"lift_point\": \"[0.8212, 0.6184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9288]\", \"lift_point\": \"[0.058, 0.9288]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.9312]\", \"lift_point\": \"[0.0576, 0.9312]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4727_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8212, 0.6184]\", \"lift_point\": \"[0.8212, 0.6184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9288]\", \"lift_point\": \"[0.058, 0.9288]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.9312]\", \"lift_point\": \"[0.0576, 0.9312]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9427]\", \"lift_point\": \"[0.058, 0.9427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4727_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8212, 0.6184]\", \"lift_point\": \"[0.8212, 0.6184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9288]\", \"lift_point\": \"[0.058, 0.9288]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.9312]\", \"lift_point\": \"[0.0576, 0.9312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9427]\", \"lift_point\": \"[0.058, 0.9427]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2302, 0.6831]\", \"lift_point\": \"[0.2302, 0.6831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4727_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8212, 0.6184]\", \"lift_point\": \"[0.8212, 0.6184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9288]\", \"lift_point\": \"[0.058, 0.9288]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.9312]\", \"lift_point\": \"[0.0576, 0.9312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9427]\", \"lift_point\": \"[0.058, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2302, 0.6831]\", \"lift_point\": \"[0.2302, 0.6831]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4727_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4800_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8274, 0.6273]\", \"lift_point\": \"[0.8274, 0.6273]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4800_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8274, 0.6273]\", \"lift_point\": \"[0.8274, 0.6273]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.3492]\", \"lift_point\": \"[0.0619, 0.3492]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4800_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8274, 0.6273]\", \"lift_point\": \"[0.8274, 0.6273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.3492]\", \"lift_point\": \"[0.0619, 0.3492]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Atlanta\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4800_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8274, 0.6273]\", \"lift_point\": \"[0.8274, 0.6273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.3492]\", \"lift_point\": \"[0.0619, 0.3492]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Atlanta\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.2697]\", \"lift_point\": \"[0.1215, 0.2697]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4800_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8274, 0.6273]\", \"lift_point\": \"[0.8274, 0.6273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.3492]\", \"lift_point\": \"[0.0619, 0.3492]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.2697]\", \"lift_point\": \"[0.1215, 0.2697]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4800_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4905_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4905_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4745, 0.1617]\", \"lift_point\": \"[0.4745, 0.1617]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4905_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4745, 0.1617]\", \"lift_point\": \"[0.4745, 0.1617]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5957, 0.4706]\", \"lift_point\": \"[0.5957, 0.4706]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4905_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4745, 0.1617]\", \"lift_point\": \"[0.4745, 0.1617]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5957, 0.4706]\", \"lift_point\": \"[0.5957, 0.4706]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/4905_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.794, 0.6887]\", \"lift_point\": \"[0.794, 0.6887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.794, 0.6887]\", \"lift_point\": \"[0.794, 0.6887]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5818]\", \"lift_point\": \"[0.0597, 0.5818]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.794, 0.6887]\", \"lift_point\": \"[0.794, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5818]\", \"lift_point\": \"[0.0597, 0.5818]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.794, 0.6887]\", \"lift_point\": \"[0.794, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5818]\", \"lift_point\": \"[0.0597, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot?\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"efrigerators at Home Depot\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.794, 0.6887]\", \"lift_point\": \"[0.794, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5818]\", \"lift_point\": \"[0.0597, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"efrigerators at Home Depot\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.5107]\", \"lift_point\": \"[0.133, 0.5107]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.794, 0.6887]\", \"lift_point\": \"[0.794, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5818]\", \"lift_point\": \"[0.0597, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"efrigerators at Home Depot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.5107]\", \"lift_point\": \"[0.133, 0.5107]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4154, 0.5376]\", \"lift_point\": \"[0.4154, 0.5376]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.6852]\", \"lift_point\": \"[0.7906, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.794, 0.6887]\", \"lift_point\": \"[0.794, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.5818]\", \"lift_point\": \"[0.0597, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"efrigerators at Home Depot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.5107]\", \"lift_point\": \"[0.133, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4154, 0.5376]\", \"lift_point\": \"[0.4154, 0.5376]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5053_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.6877]\", \"lift_point\": \"[0.7884, 0.6877]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.6877]\", \"lift_point\": \"[0.7884, 0.6877]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7929, 0.6916]\", \"lift_point\": \"[0.7929, 0.6916]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.6877]\", \"lift_point\": \"[0.7884, 0.6877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7929, 0.6916]\", \"lift_point\": \"[0.7929, 0.6916]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.648]\", \"lift_point\": \"[0.064, 0.648]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.6877]\", \"lift_point\": \"[0.7884, 0.6877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7929, 0.6916]\", \"lift_point\": \"[0.7929, 0.6916]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.648]\", \"lift_point\": \"[0.064, 0.648]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9276]\", \"lift_point\": \"[0.0588, 0.9276]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.6877]\", \"lift_point\": \"[0.7884, 0.6877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7929, 0.6916]\", \"lift_point\": \"[0.7929, 0.6916]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.648]\", \"lift_point\": \"[0.064, 0.648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9276]\", \"lift_point\": \"[0.0588, 0.9276]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotel in Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.6877]\", \"lift_point\": \"[0.7884, 0.6877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7929, 0.6916]\", \"lift_point\": \"[0.7929, 0.6916]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.648]\", \"lift_point\": \"[0.064, 0.648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9276]\", \"lift_point\": \"[0.0588, 0.9276]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotel in Buenos aires\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1293, 0.501]\", \"lift_point\": \"[0.1293, 0.501]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.6877]\", \"lift_point\": \"[0.7884, 0.6877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7929, 0.6916]\", \"lift_point\": \"[0.7929, 0.6916]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.648]\", \"lift_point\": \"[0.064, 0.648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9276]\", \"lift_point\": \"[0.0588, 0.9276]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotel in Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1293, 0.501]\", \"lift_point\": \"[0.1293, 0.501]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5067_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7899, 0.6865]\", \"lift_point\": \"[0.7899, 0.6865]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5133_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7899, 0.6865]\", \"lift_point\": \"[0.7899, 0.6865]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5763]\", \"lift_point\": \"[0.0555, 0.5763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5133_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7899, 0.6865]\", \"lift_point\": \"[0.7899, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5763]\", \"lift_point\": \"[0.0555, 0.5763]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.9302]\", \"lift_point\": \"[0.0608, 0.9302]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5133_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7899, 0.6865]\", \"lift_point\": \"[0.7899, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5763]\", \"lift_point\": \"[0.0555, 0.5763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.9302]\", \"lift_point\": \"[0.0608, 0.9302]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5133_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7899, 0.6865]\", \"lift_point\": \"[0.7899, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5763]\", \"lift_point\": \"[0.0555, 0.5763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.9302]\", \"lift_point\": \"[0.0608, 0.9302]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1225, 0.4475]\", \"lift_point\": \"[0.1225, 0.4475]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5133_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7899, 0.6865]\", \"lift_point\": \"[0.7899, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.5763]\", \"lift_point\": \"[0.0555, 0.5763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.9302]\", \"lift_point\": \"[0.0608, 0.9302]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1225, 0.4475]\", \"lift_point\": \"[0.1225, 0.4475]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5133_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6914]\", \"lift_point\": \"[0.7887, 0.6914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5194_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6914]\", \"lift_point\": \"[0.7887, 0.6914]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4808]\", \"lift_point\": \"[0.0686, 0.4808]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5194_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6914]\", \"lift_point\": \"[0.7887, 0.6914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4808]\", \"lift_point\": \"[0.0686, 0.4808]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9312]\", \"lift_point\": \"[0.0561, 0.9312]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5194_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6914]\", \"lift_point\": \"[0.7887, 0.6914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4808]\", \"lift_point\": \"[0.0686, 0.4808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9312]\", \"lift_point\": \"[0.0561, 0.9312]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Colombia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5194_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6914]\", \"lift_point\": \"[0.7887, 0.6914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4808]\", \"lift_point\": \"[0.0686, 0.4808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9312]\", \"lift_point\": \"[0.0561, 0.9312]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Colombia\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.3141]\", \"lift_point\": \"[0.1197, 0.3141]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5194_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6914]\", \"lift_point\": \"[0.7887, 0.6914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4808]\", \"lift_point\": \"[0.0686, 0.4808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9312]\", \"lift_point\": \"[0.0561, 0.9312]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Colombia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.3141]\", \"lift_point\": \"[0.1197, 0.3141]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5194_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.688]\", \"lift_point\": \"[0.7875, 0.688]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5216_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.688]\", \"lift_point\": \"[0.7875, 0.688]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.7198]\", \"lift_point\": \"[0.0648, 0.7198]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5216_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.688]\", \"lift_point\": \"[0.7875, 0.688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.7198]\", \"lift_point\": \"[0.0648, 0.7198]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9508]\", \"lift_point\": \"[0.0563, 0.9508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5216_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.688]\", \"lift_point\": \"[0.7875, 0.688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.7198]\", \"lift_point\": \"[0.0648, 0.7198]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9508]\", \"lift_point\": \"[0.0563, 0.9508]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5745]\", \"lift_point\": \"[0.048, 0.5745]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5216_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.688]\", \"lift_point\": \"[0.7875, 0.688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.7198]\", \"lift_point\": \"[0.0648, 0.7198]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9508]\", \"lift_point\": \"[0.0563, 0.9508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5745]\", \"lift_point\": \"[0.048, 0.5745]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5216_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.688]\", \"lift_point\": \"[0.7875, 0.688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.7198]\", \"lift_point\": \"[0.0648, 0.7198]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9508]\", \"lift_point\": \"[0.0563, 0.9508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5745]\", \"lift_point\": \"[0.048, 0.5745]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Paris\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1206, 0.5815]\", \"lift_point\": \"[0.1206, 0.5815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5216_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.688]\", \"lift_point\": \"[0.7875, 0.688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.7198]\", \"lift_point\": \"[0.0648, 0.7198]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9508]\", \"lift_point\": \"[0.0563, 0.9508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5745]\", \"lift_point\": \"[0.048, 0.5745]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1206, 0.5815]\", \"lift_point\": \"[0.1206, 0.5815]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5216_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Uber app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9466]\", \"lift_point\": \"[0.0551, 0.9466]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9466]\", \"lift_point\": \"[0.0551, 0.9466]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"uber app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9466]\", \"lift_point\": \"[0.0551, 0.9466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"uber app\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1251, 0.2961]\", \"lift_point\": \"[0.1251, 0.2961]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9466]\", \"lift_point\": \"[0.0551, 0.9466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1251, 0.2961]\", \"lift_point\": \"[0.1251, 0.2961]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.8404]\", \"lift_point\": \"[0.1433, 0.8404]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9466]\", \"lift_point\": \"[0.0551, 0.9466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1251, 0.2961]\", \"lift_point\": \"[0.1251, 0.2961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.8404]\", \"lift_point\": \"[0.1433, 0.8404]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.8404]\", \"lift_point\": \"[0.1433, 0.8404]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9466]\", \"lift_point\": \"[0.0551, 0.9466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1251, 0.2961]\", \"lift_point\": \"[0.1251, 0.2961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.8404]\", \"lift_point\": \"[0.1433, 0.8404]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.8404]\", \"lift_point\": \"[0.1433, 0.8404]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5041, 0.5143]\", \"lift_point\": \"[0.5041, 0.5143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7856, 0.5152]\", \"lift_point\": \"[0.7856, 0.5152]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8038]\", \"lift_point\": \"[0.0519, 0.8038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9466]\", \"lift_point\": \"[0.0551, 0.9466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1251, 0.2961]\", \"lift_point\": \"[0.1251, 0.2961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.8404]\", \"lift_point\": \"[0.1433, 0.8404]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.8404]\", \"lift_point\": \"[0.1433, 0.8404]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5041, 0.5143]\", \"lift_point\": \"[0.5041, 0.5143]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5281_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.4462]\", \"lift_point\": \"[0.063, 0.4462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.4462]\", \"lift_point\": \"[0.063, 0.4462]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9393]\", \"lift_point\": \"[0.0613, 0.9393]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.4462]\", \"lift_point\": \"[0.063, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9393]\", \"lift_point\": \"[0.0613, 0.9393]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what is the speed of a jet\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.4462]\", \"lift_point\": \"[0.063, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9393]\", \"lift_point\": \"[0.0613, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what is the speed of a jet\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.479]\", \"lift_point\": \"[0.1164, 0.479]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.4462]\", \"lift_point\": \"[0.063, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9393]\", \"lift_point\": \"[0.0613, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what is the speed of a jet\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.479]\", \"lift_point\": \"[0.1164, 0.479]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.246]\", \"lift_point\": \"[0.3651, 0.246]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8037, 0.717]\", \"lift_point\": \"[0.8037, 0.717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.4462]\", \"lift_point\": \"[0.063, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9393]\", \"lift_point\": \"[0.0613, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what is the speed of a jet\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.479]\", \"lift_point\": \"[0.1164, 0.479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.246]\", \"lift_point\": \"[0.3651, 0.246]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5312_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.6164]\", \"lift_point\": \"[0.0572, 0.6164]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.6164]\", \"lift_point\": \"[0.0572, 0.6164]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.9258]\", \"lift_point\": \"[0.0602, 0.9258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.6164]\", \"lift_point\": \"[0.0572, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.9258]\", \"lift_point\": \"[0.0602, 0.9258]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach blanket\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.6164]\", \"lift_point\": \"[0.0572, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.9258]\", \"lift_point\": \"[0.0602, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach blanket\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.6164]\", \"lift_point\": \"[0.0572, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.9258]\", \"lift_point\": \"[0.0602, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach blanket\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1209, 0.6377]\", \"lift_point\": \"[0.1209, 0.6377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.6164]\", \"lift_point\": \"[0.0572, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.9258]\", \"lift_point\": \"[0.0602, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach blanket\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1209, 0.6377]\", \"lift_point\": \"[0.1209, 0.6377]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4363, 0.4836]\", \"lift_point\": \"[0.4363, 0.4836]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7995, 0.6924]\", \"lift_point\": \"[0.7995, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.6164]\", \"lift_point\": \"[0.0572, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.9258]\", \"lift_point\": \"[0.0602, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach blanket\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1209, 0.6377]\", \"lift_point\": \"[0.1209, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4363, 0.4836]\", \"lift_point\": \"[0.4363, 0.4836]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5403_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5410_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7954, 0.726]\", \"lift_point\": \"[0.7954, 0.726]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5410_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7954, 0.726]\", \"lift_point\": \"[0.7954, 0.726]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.5287]\", \"lift_point\": \"[0.055, 0.5287]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5410_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7954, 0.726]\", \"lift_point\": \"[0.7954, 0.726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.5287]\", \"lift_point\": \"[0.055, 0.5287]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9427]\", \"lift_point\": \"[0.0638, 0.9427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5410_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7954, 0.726]\", \"lift_point\": \"[0.7954, 0.726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.5287]\", \"lift_point\": \"[0.055, 0.5287]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9427]\", \"lift_point\": \"[0.0638, 0.9427]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of brazil\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5410_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7954, 0.726]\", \"lift_point\": \"[0.7954, 0.726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.5287]\", \"lift_point\": \"[0.055, 0.5287]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9427]\", \"lift_point\": \"[0.0638, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of brazil\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.2331]\", \"lift_point\": \"[0.1306, 0.2331]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5410_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7954, 0.726]\", \"lift_point\": \"[0.7954, 0.726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.5287]\", \"lift_point\": \"[0.055, 0.5287]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9427]\", \"lift_point\": \"[0.0638, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of brazil\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.2331]\", \"lift_point\": \"[0.1306, 0.2331]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5410_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.5349]\", \"lift_point\": \"[0.0571, 0.5349]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.5349]\", \"lift_point\": \"[0.0571, 0.5349]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9384]\", \"lift_point\": \"[0.0569, 0.9384]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.5349]\", \"lift_point\": \"[0.0571, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9384]\", \"lift_point\": \"[0.0569, 0.9384]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in barbados\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.5349]\", \"lift_point\": \"[0.0571, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9384]\", \"lift_point\": \"[0.0569, 0.9384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in barbados\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.5349]\", \"lift_point\": \"[0.0571, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9384]\", \"lift_point\": \"[0.0569, 0.9384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in barbados\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.2067]\", \"lift_point\": \"[0.1341, 0.2067]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.5349]\", \"lift_point\": \"[0.0571, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9384]\", \"lift_point\": \"[0.0569, 0.9384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in barbados\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.2067]\", \"lift_point\": \"[0.1341, 0.2067]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5863, 0.3413]\", \"lift_point\": \"[0.5863, 0.3413]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6743]\", \"lift_point\": \"[0.7883, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.5349]\", \"lift_point\": \"[0.0571, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.9384]\", \"lift_point\": \"[0.0569, 0.9384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in barbados\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.2067]\", \"lift_point\": \"[0.1341, 0.2067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5863, 0.3413]\", \"lift_point\": \"[0.5863, 0.3413]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5425_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.833]\", \"lift_point\": \"[0.778, 0.833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.619, 0.4869]\", \"lift_point\": \"[0.619, 0.4869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3443, 0.3518]\", \"lift_point\": \"[0.3443, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2702, 0.36]\", \"lift_point\": \"[0.2702, 0.36]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.39, 0.2406]\", \"lift_point\": \"[0.39, 0.2406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7814, 0.9053]\", \"lift_point\": \"[0.7814, 0.9053]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7814, 0.9053]\", \"lift_point\": \"[0.7814, 0.9053]\", \"typed_text\": \"\" Goal: Toggle notifications for the Google Maps app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5713_18.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1654, 0.1646]\", \"lift_point\": \"[0.1654, 0.1646]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1654, 0.1646]\", \"lift_point\": \"[0.1654, 0.1646]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.7892]\", \"lift_point\": \"[0.0629, 0.7892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1654, 0.1646]\", \"lift_point\": \"[0.1654, 0.1646]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.7892]\", \"lift_point\": \"[0.0629, 0.7892]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1654, 0.1646]\", \"lift_point\": \"[0.1654, 0.1646]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.7892]\", \"lift_point\": \"[0.0629, 0.7892]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0753, 0.2557]\", \"lift_point\": \"[0.0753, 0.2557]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1654, 0.1646]\", \"lift_point\": \"[0.1654, 0.1646]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.7892]\", \"lift_point\": \"[0.0629, 0.7892]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0753, 0.2557]\", \"lift_point\": \"[0.0753, 0.2557]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Good french restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1654, 0.1646]\", \"lift_point\": \"[0.1654, 0.1646]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.7892]\", \"lift_point\": \"[0.0629, 0.7892]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0753, 0.2557]\", \"lift_point\": \"[0.0753, 0.2557]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Good french restaurants\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1893, 0.4882]\", \"lift_point\": \"[0.1893, 0.4882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1654, 0.1646]\", \"lift_point\": \"[0.1654, 0.1646]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0629, 0.7892]\", \"lift_point\": \"[0.0629, 0.7892]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0753, 0.2557]\", \"lift_point\": \"[0.0753, 0.2557]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Good french restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1893, 0.4882]\", \"lift_point\": \"[0.1893, 0.4882]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5805_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.8954]\", \"lift_point\": \"[0.0714, 0.8954]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.8954]\", \"lift_point\": \"[0.0714, 0.8954]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.2223]\", \"lift_point\": \"[0.05, 0.2223]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.8954]\", \"lift_point\": \"[0.0714, 0.8954]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.2223]\", \"lift_point\": \"[0.05, 0.2223]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.8954]\", \"lift_point\": \"[0.0714, 0.8954]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.2223]\", \"lift_point\": \"[0.05, 0.2223]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1224, 0.4455]\", \"lift_point\": \"[0.1224, 0.4455]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.4552]\", \"lift_point\": \"[0.8789, 0.4552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.412]\", \"lift_point\": \"[0.8867, 0.412]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.8954]\", \"lift_point\": \"[0.0714, 0.8954]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.2223]\", \"lift_point\": \"[0.05, 0.2223]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Macys?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1224, 0.4455]\", \"lift_point\": \"[0.1224, 0.4455]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/5905_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6012_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6012_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.908, 0.4306]\", \"lift_point\": \"[0.908, 0.4306]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.908, 0.4306]\", \"lift_point\": \"[0.908, 0.4306]\", \"typed_text\": \"\" Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.2757]\", \"lift_point\": \"[0.0449, 0.2757]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.908, 0.4306]\", \"lift_point\": \"[0.908, 0.4306]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.2757]\", \"lift_point\": \"[0.0449, 0.2757]\", \"typed_text\": \"\" Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.908, 0.4306]\", \"lift_point\": \"[0.908, 0.4306]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.2757]\", \"lift_point\": \"[0.0449, 0.2757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\" Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the prime minister of Canada?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.908, 0.4306]\", \"lift_point\": \"[0.908, 0.4306]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.2757]\", \"lift_point\": \"[0.0449, 0.2757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the prime minister of Canada?\" Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.908, 0.4306]\", \"lift_point\": \"[0.908, 0.4306]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.2757]\", \"lift_point\": \"[0.0449, 0.2757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the prime minister of Canada?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\" Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1254, 0.4286]\", \"lift_point\": \"[0.1254, 0.4286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.908, 0.4306]\", \"lift_point\": \"[0.908, 0.4306]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.2757]\", \"lift_point\": \"[0.0449, 0.2757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the prime minister of Canada?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6864, 0.7784]\", \"lift_point\": \"[0.6864, 0.7784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1254, 0.4286]\", \"lift_point\": \"[0.1254, 0.4286]\", \"typed_text\": \"\" Goal: Who is the prime minister of Canada?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6099_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8022, 0.8505]\", \"lift_point\": \"[0.8022, 0.8505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.2629]\", \"lift_point\": \"[0.1171, 0.2629]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7986, 0.8009]\", \"lift_point\": \"[0.7986, 0.8009]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.2629]\", \"lift_point\": \"[0.1171, 0.2629]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.232, 0.2437]\", \"lift_point\": \"[0.232, 0.2437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0611]\", \"lift_point\": \"[0.0543, 0.0611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.2629]\", \"lift_point\": \"[0.1171, 0.2629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.232, 0.2437]\", \"lift_point\": \"[0.232, 0.2437]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.241, 0.2248]\", \"lift_point\": \"[0.241, 0.2248]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.045, 0.8247]\", \"lift_point\": \"[0.045, 0.8247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.2629]\", \"lift_point\": \"[0.1171, 0.2629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.232, 0.2437]\", \"lift_point\": \"[0.232, 0.2437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.241, 0.2248]\", \"lift_point\": \"[0.241, 0.2248]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2338, 0.1303]\", \"lift_point\": \"[0.2338, 0.1303]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.724]\", \"lift_point\": \"[0.0651, 0.724]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.2629]\", \"lift_point\": \"[0.1171, 0.2629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.232, 0.2437]\", \"lift_point\": \"[0.232, 0.2437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.241, 0.2248]\", \"lift_point\": \"[0.241, 0.2248]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2338, 0.1303]\", \"lift_point\": \"[0.2338, 0.1303]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1083, 0.5662]\", \"lift_point\": \"[0.1083, 0.5662]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign lates video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.2629]\", \"lift_point\": \"[0.1171, 0.2629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.232, 0.2437]\", \"lift_point\": \"[0.232, 0.2437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.241, 0.2248]\", \"lift_point\": \"[0.241, 0.2248]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2338, 0.1303]\", \"lift_point\": \"[0.2338, 0.1303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1083, 0.5662]\", \"lift_point\": \"[0.1083, 0.5662]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1475, 0.5133]\", \"lift_point\": \"[0.1475, 0.5133]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.3189]\", \"lift_point\": \"[0.1314, 0.3189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.7127]\", \"lift_point\": \"[0.1426, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.2629]\", \"lift_point\": \"[0.1171, 0.2629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.232, 0.2437]\", \"lift_point\": \"[0.232, 0.2437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.241, 0.2248]\", \"lift_point\": \"[0.241, 0.2248]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2338, 0.1303]\", \"lift_point\": \"[0.2338, 0.1303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1083, 0.5662]\", \"lift_point\": \"[0.1083, 0.5662]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1475, 0.5133]\", \"lift_point\": \"[0.1475, 0.5133]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6116_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the files app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6170_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the files app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5251, 0.5935]\", \"lift_point\": \"[0.5251, 0.5935]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6170_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5251, 0.5935]\", \"lift_point\": \"[0.5251, 0.5935]\", \"typed_text\": \"\" Goal: Open the files app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6170_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.4438]\", \"lift_point\": \"[0.8857, 0.4438]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6471_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.4438]\", \"lift_point\": \"[0.8857, 0.4438]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.3127]\", \"lift_point\": \"[0.067, 0.3127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6471_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.4438]\", \"lift_point\": \"[0.8857, 0.4438]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.3127]\", \"lift_point\": \"[0.067, 0.3127]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6471_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.4438]\", \"lift_point\": \"[0.8857, 0.4438]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.3127]\", \"lift_point\": \"[0.067, 0.3127]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1437, 0.4795]\", \"lift_point\": \"[0.1437, 0.4795]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6471_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.4438]\", \"lift_point\": \"[0.8857, 0.4438]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.3127]\", \"lift_point\": \"[0.067, 0.3127]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1437, 0.4795]\", \"lift_point\": \"[0.1437, 0.4795]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6471_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6549_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.1003]\", \"lift_point\": \"[0.7989, 0.1003]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6549_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.1003]\", \"lift_point\": \"[0.7989, 0.1003]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6549_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Taylor Swift video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Taylor Swift video\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.4143]\", \"lift_point\": \"[0.1215, 0.4143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Taylor Swift video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.4143]\", \"lift_point\": \"[0.1215, 0.4143]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Taylor Swift video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.4143]\", \"lift_point\": \"[0.1215, 0.4143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.8909]\", \"lift_point\": \"[0.7828, 0.8909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Taylor Swift video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.4143]\", \"lift_point\": \"[0.1215, 0.4143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6668, 0.4068]\", \"lift_point\": \"[0.6668, 0.4068]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.0588]\", \"lift_point\": \"[0.0677, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Taylor Swift video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.4143]\", \"lift_point\": \"[0.1215, 0.4143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6668, 0.4068]\", \"lift_point\": \"[0.6668, 0.4068]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1596, 0.4751]\", \"lift_point\": \"[0.1596, 0.4751]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.8052]\", \"lift_point\": \"[0.0564, 0.8052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.5065]\", \"lift_point\": \"[0.0675, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Taylor Swift video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.4143]\", \"lift_point\": \"[0.1215, 0.4143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6668, 0.4068]\", \"lift_point\": \"[0.6668, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1596, 0.4751]\", \"lift_point\": \"[0.1596, 0.4751]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6741_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.517]\", \"lift_point\": \"[0.8836, 0.517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.517]\", \"lift_point\": \"[0.8836, 0.517]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.3216]\", \"lift_point\": \"[0.0591, 0.3216]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.517]\", \"lift_point\": \"[0.8836, 0.517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.3216]\", \"lift_point\": \"[0.0591, 0.3216]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.517]\", \"lift_point\": \"[0.8836, 0.517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.3216]\", \"lift_point\": \"[0.0591, 0.3216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds?\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.4823]\", \"lift_point\": \"[0.1436, 0.4823]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.517]\", \"lift_point\": \"[0.8836, 0.517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.3216]\", \"lift_point\": \"[0.0591, 0.3216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.4823]\", \"lift_point\": \"[0.1436, 0.4823]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.7515]\", \"lift_point\": \"[0.5388, 0.7515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.517]\", \"lift_point\": \"[0.8836, 0.517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.3216]\", \"lift_point\": \"[0.0591, 0.3216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.4823]\", \"lift_point\": \"[0.1436, 0.4823]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.7515]\", \"lift_point\": \"[0.5388, 0.7515]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.259]\", \"lift_point\": \"[0.1429, 0.259]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.517]\", \"lift_point\": \"[0.8836, 0.517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.3216]\", \"lift_point\": \"[0.0591, 0.3216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.4823]\", \"lift_point\": \"[0.1436, 0.4823]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.7515]\", \"lift_point\": \"[0.5388, 0.7515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.259]\", \"lift_point\": \"[0.1429, 0.259]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6841_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on Reddit this week>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6865_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.4192]\", \"lift_point\": \"[0.8912, 0.4192]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6865_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.4192]\", \"lift_point\": \"[0.8912, 0.4192]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.2979]\", \"lift_point\": \"[0.0727, 0.2979]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6865_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.4192]\", \"lift_point\": \"[0.8912, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.2979]\", \"lift_point\": \"[0.0727, 0.2979]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6865_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.4192]\", \"lift_point\": \"[0.8912, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.2979]\", \"lift_point\": \"[0.0727, 0.2979]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4813, 0.3744]\", \"lift_point\": \"[0.4813, 0.3744]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6865_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.4192]\", \"lift_point\": \"[0.8912, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.2979]\", \"lift_point\": \"[0.0727, 0.2979]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4813, 0.3744]\", \"lift_point\": \"[0.4813, 0.3744]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.36, 0.3085]\", \"lift_point\": \"[0.36, 0.3085]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6865_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.4192]\", \"lift_point\": \"[0.8912, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.2979]\", \"lift_point\": \"[0.0727, 0.2979]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4813, 0.3744]\", \"lift_point\": \"[0.4813, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.36, 0.3085]\", \"lift_point\": \"[0.36, 0.3085]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6865_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.3955]\", \"lift_point\": \"[0.8819, 0.3955]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.3955]\", \"lift_point\": \"[0.8819, 0.3955]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.7775]\", \"lift_point\": \"[0.8824, 0.7775]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.3955]\", \"lift_point\": \"[0.8819, 0.3955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.7775]\", \"lift_point\": \"[0.8824, 0.7775]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.8149]\", \"lift_point\": \"[0.8924, 0.8149]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.3955]\", \"lift_point\": \"[0.8819, 0.3955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.7775]\", \"lift_point\": \"[0.8824, 0.7775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.8149]\", \"lift_point\": \"[0.8924, 0.8149]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.5462]\", \"lift_point\": \"[0.3153, 0.5462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.3955]\", \"lift_point\": \"[0.8819, 0.3955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.7775]\", \"lift_point\": \"[0.8824, 0.7775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.8149]\", \"lift_point\": \"[0.8924, 0.8149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.5462]\", \"lift_point\": \"[0.3153, 0.5462]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in San Francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.3955]\", \"lift_point\": \"[0.8819, 0.3955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.7775]\", \"lift_point\": \"[0.8824, 0.7775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.8149]\", \"lift_point\": \"[0.8924, 0.8149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.5462]\", \"lift_point\": \"[0.3153, 0.5462]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in San Francisco\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.6766]\", \"lift_point\": \"[0.1419, 0.6766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.6865]\", \"lift_point\": \"[0.7714, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.3955]\", \"lift_point\": \"[0.8819, 0.3955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.7775]\", \"lift_point\": \"[0.8824, 0.7775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.8149]\", \"lift_point\": \"[0.8924, 0.8149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.5462]\", \"lift_point\": \"[0.3153, 0.5462]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in San Francisco\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.6766]\", \"lift_point\": \"[0.1419, 0.6766]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/6908_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7038_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9109, 0.4037]\", \"lift_point\": \"[0.9109, 0.4037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7038_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9109, 0.4037]\", \"lift_point\": \"[0.9109, 0.4037]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Olive Garden?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7038_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9109, 0.4037]\", \"lift_point\": \"[0.9109, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Olive Garden?\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2294]\", \"lift_point\": \"[0.1182, 0.2294]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7038_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9109, 0.4037]\", \"lift_point\": \"[0.9109, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Olive Garden?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2294]\", \"lift_point\": \"[0.1182, 0.2294]\", \"typed_text\": \"\" Goal: What's on the menu at Olive Garden?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7038_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7051_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7975, 0.8581]\", \"lift_point\": \"[0.7975, 0.8581]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7051_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7975, 0.8581]\", \"lift_point\": \"[0.7975, 0.8581]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.7968]\", \"lift_point\": \"[0.0609, 0.7968]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7051_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7975, 0.8581]\", \"lift_point\": \"[0.7975, 0.8581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.7968]\", \"lift_point\": \"[0.0609, 0.7968]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bruno Mars\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7051_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7975, 0.8581]\", \"lift_point\": \"[0.7975, 0.8581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.7968]\", \"lift_point\": \"[0.0609, 0.7968]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bruno Mars\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1004, 0.2786]\", \"lift_point\": \"[0.1004, 0.2786]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7051_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7975, 0.8581]\", \"lift_point\": \"[0.7975, 0.8581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.7968]\", \"lift_point\": \"[0.0609, 0.7968]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bruno Mars\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1004, 0.2786]\", \"lift_point\": \"[0.1004, 0.2786]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4055, 0.627]\", \"lift_point\": \"[0.4055, 0.627]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7051_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7975, 0.8581]\", \"lift_point\": \"[0.7975, 0.8581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.7968]\", \"lift_point\": \"[0.0609, 0.7968]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bruno Mars\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1004, 0.2786]\", \"lift_point\": \"[0.1004, 0.2786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4055, 0.627]\", \"lift_point\": \"[0.4055, 0.627]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7051_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [STATUS_TASK_IMPOSSIBLE]\n; Action Decision: \"action_type\": \"STATUS_TASK_IMPOSSIBLE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7065_1.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7177_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7177_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4996, 0.1605]\", \"lift_point\": \"[0.4996, 0.1605]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7177_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4996, 0.1605]\", \"lift_point\": \"[0.4996, 0.1605]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.491, 0.2181]\", \"lift_point\": \"[0.491, 0.2181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7177_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4996, 0.1605]\", \"lift_point\": \"[0.4996, 0.1605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.491, 0.2181]\", \"lift_point\": \"[0.491, 0.2181]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7177_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new blush>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7237_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.347]\", \"lift_point\": \"[0.9053, 0.347]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7237_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.347]\", \"lift_point\": \"[0.9053, 0.347]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new blush\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7237_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.347]\", \"lift_point\": \"[0.9053, 0.347]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new blush\" Goal: Search for a new blush>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1311, 0.364]\", \"lift_point\": \"[0.1311, 0.364]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7237_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.347]\", \"lift_point\": \"[0.9053, 0.347]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new blush\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1311, 0.364]\", \"lift_point\": \"[0.1311, 0.364]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7237_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8768, 0.4786]\", \"lift_point\": \"[0.8768, 0.4786]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7685_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8768, 0.4786]\", \"lift_point\": \"[0.8768, 0.4786]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7685_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8768, 0.4786]\", \"lift_point\": \"[0.8768, 0.4786]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7685_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8768, 0.4786]\", \"lift_point\": \"[0.8768, 0.4786]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1938, 0.5989]\", \"lift_point\": \"[0.1938, 0.5989]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7685_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8768, 0.4786]\", \"lift_point\": \"[0.8768, 0.4786]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1938, 0.5989]\", \"lift_point\": \"[0.1938, 0.5989]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7685_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3606]\", \"lift_point\": \"[0.123, 0.3606]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3606]\", \"lift_point\": \"[0.123, 0.3606]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6627, 0.8258]\", \"lift_point\": \"[0.6627, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3606]\", \"lift_point\": \"[0.123, 0.3606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.052]\", \"lift_point\": \"[0.0549, 0.052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3606]\", \"lift_point\": \"[0.123, 0.3606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4453, 0.3949]\", \"lift_point\": \"[0.4453, 0.3949]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0547]\", \"lift_point\": \"[0.0515, 0.0547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3606]\", \"lift_point\": \"[0.123, 0.3606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4453, 0.3949]\", \"lift_point\": \"[0.4453, 0.3949]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1563, 0.5084]\", \"lift_point\": \"[0.1563, 0.5084]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.8045]\", \"lift_point\": \"[0.0554, 0.8045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3606]\", \"lift_point\": \"[0.123, 0.3606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4453, 0.3949]\", \"lift_point\": \"[0.4453, 0.3949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1563, 0.5084]\", \"lift_point\": \"[0.1563, 0.5084]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3141, 0.9451]\", \"lift_point\": \"[0.3141, 0.9451]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Game Informer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3606]\", \"lift_point\": \"[0.123, 0.3606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4453, 0.3949]\", \"lift_point\": \"[0.4453, 0.3949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1563, 0.5084]\", \"lift_point\": \"[0.1563, 0.5084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3141, 0.9451]\", \"lift_point\": \"[0.3141, 0.9451]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7705_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.9156]\", \"lift_point\": \"[0.0586, 0.9156]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.9156]\", \"lift_point\": \"[0.0586, 0.9156]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.9156]\", \"lift_point\": \"[0.0586, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2572, 0.3211]\", \"lift_point\": \"[0.2572, 0.3211]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.9156]\", \"lift_point\": \"[0.0586, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2572, 0.3211]\", \"lift_point\": \"[0.2572, 0.3211]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8651, 0.8292]\", \"lift_point\": \"[0.8651, 0.8292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5595, 0.1967]\", \"lift_point\": \"[0.5595, 0.1967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.5747]\", \"lift_point\": \"[0.8946, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit this week\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.9156]\", \"lift_point\": \"[0.0586, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2572, 0.3211]\", \"lift_point\": \"[0.2572, 0.3211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8651, 0.8292]\", \"lift_point\": \"[0.8651, 0.8292]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/7864_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in London>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8127_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.635]\", \"lift_point\": \"[0.7825, 0.635]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8127_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.635]\", \"lift_point\": \"[0.7825, 0.635]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.557]\", \"lift_point\": \"[0.0497, 0.557]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8127_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.635]\", \"lift_point\": \"[0.7825, 0.635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.557]\", \"lift_point\": \"[0.0497, 0.557]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8127_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.635]\", \"lift_point\": \"[0.7825, 0.635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.557]\", \"lift_point\": \"[0.0497, 0.557]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1048, 0.2982]\", \"lift_point\": \"[0.1048, 0.2982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8127_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.635]\", \"lift_point\": \"[0.7825, 0.635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.557]\", \"lift_point\": \"[0.0497, 0.557]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1048, 0.2982]\", \"lift_point\": \"[0.1048, 0.2982]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8127_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US president?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8270_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US president?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6244]\", \"lift_point\": \"[0.7789, 0.6244]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8270_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6244]\", \"lift_point\": \"[0.7789, 0.6244]\", \"typed_text\": \"\" Goal: What's the news about the US president?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0431, 0.5396]\", \"lift_point\": \"[0.0431, 0.5396]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8270_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6244]\", \"lift_point\": \"[0.7789, 0.6244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0431, 0.5396]\", \"lift_point\": \"[0.0431, 0.5396]\", \"typed_text\": \"\" Goal: What's the news about the US president?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US president?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8270_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6244]\", \"lift_point\": \"[0.7789, 0.6244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0431, 0.5396]\", \"lift_point\": \"[0.0431, 0.5396]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US president?\" Goal: What's the news about the US president?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.119, 0.822]\", \"lift_point\": \"[0.119, 0.822]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8270_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6244]\", \"lift_point\": \"[0.7789, 0.6244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0431, 0.5396]\", \"lift_point\": \"[0.0431, 0.5396]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US president?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.119, 0.822]\", \"lift_point\": \"[0.119, 0.822]\", \"typed_text\": \"\" Goal: What's the news about the US president?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3067, 0.2148]\", \"lift_point\": \"[0.3067, 0.2148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8270_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6244]\", \"lift_point\": \"[0.7789, 0.6244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0431, 0.5396]\", \"lift_point\": \"[0.0431, 0.5396]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US president?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.119, 0.822]\", \"lift_point\": \"[0.119, 0.822]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3067, 0.2148]\", \"lift_point\": \"[0.3067, 0.2148]\", \"typed_text\": \"\" Goal: What's the news about the US president?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8270_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7966, 0.5681]\", \"lift_point\": \"[0.7966, 0.5681]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8280_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7966, 0.5681]\", \"lift_point\": \"[0.7966, 0.5681]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.4399]\", \"lift_point\": \"[0.0621, 0.4399]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8280_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7966, 0.5681]\", \"lift_point\": \"[0.7966, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.4399]\", \"lift_point\": \"[0.0621, 0.4399]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipgloss\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8280_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7966, 0.5681]\", \"lift_point\": \"[0.7966, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.4399]\", \"lift_point\": \"[0.0621, 0.4399]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipgloss\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1204, 0.187]\", \"lift_point\": \"[0.1204, 0.187]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8280_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7966, 0.5681]\", \"lift_point\": \"[0.7966, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.4399]\", \"lift_point\": \"[0.0621, 0.4399]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipgloss\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1204, 0.187]\", \"lift_point\": \"[0.1204, 0.187]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8280_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8939, 0.1512]\", \"lift_point\": \"[0.8939, 0.1512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8310_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8939, 0.1512]\", \"lift_point\": \"[0.8939, 0.1512]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.0816]\", \"lift_point\": \"[0.0538, 0.0816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8310_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8939, 0.1512]\", \"lift_point\": \"[0.8939, 0.1512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.0816]\", \"lift_point\": \"[0.0538, 0.0816]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8310_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this morning?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8514_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this morning?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7933, 0.6138]\", \"lift_point\": \"[0.7933, 0.6138]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8514_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7933, 0.6138]\", \"lift_point\": \"[0.7933, 0.6138]\", \"typed_text\": \"\" Goal: What's the news this morning?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2905]\", \"lift_point\": \"[0.0518, 0.2905]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8514_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7933, 0.6138]\", \"lift_point\": \"[0.7933, 0.6138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2905]\", \"lift_point\": \"[0.0518, 0.2905]\", \"typed_text\": \"\" Goal: What's the news this morning?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this morning?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8514_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7933, 0.6138]\", \"lift_point\": \"[0.7933, 0.6138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2905]\", \"lift_point\": \"[0.0518, 0.2905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this morning?\" Goal: What's the news this morning?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.7887]\", \"lift_point\": \"[0.1125, 0.7887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8514_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7933, 0.6138]\", \"lift_point\": \"[0.7933, 0.6138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2905]\", \"lift_point\": \"[0.0518, 0.2905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this morning?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.7887]\", \"lift_point\": \"[0.1125, 0.7887]\", \"typed_text\": \"\" Goal: What's the news this morning?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2198, 0.1717]\", \"lift_point\": \"[0.2198, 0.1717]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8514_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7933, 0.6138]\", \"lift_point\": \"[0.7933, 0.6138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.2905]\", \"lift_point\": \"[0.0518, 0.2905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this morning?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.7887]\", \"lift_point\": \"[0.1125, 0.7887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2198, 0.1717]\", \"lift_point\": \"[0.2198, 0.1717]\", \"typed_text\": \"\" Goal: What's the news this morning?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8514_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9547]\", \"lift_point\": \"[0.0461, 0.9547]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9547]\", \"lift_point\": \"[0.0461, 0.9547]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.0544]\", \"lift_point\": \"[0.0596, 0.0544]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9547]\", \"lift_point\": \"[0.0461, 0.9547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.0544]\", \"lift_point\": \"[0.0596, 0.0544]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.6459]\", \"lift_point\": \"[0.0442, 0.6459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9547]\", \"lift_point\": \"[0.0461, 0.9547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.0544]\", \"lift_point\": \"[0.0596, 0.0544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.6459]\", \"lift_point\": \"[0.0442, 0.6459]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Katy Perry video on YouTube\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9547]\", \"lift_point\": \"[0.0461, 0.9547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.0544]\", \"lift_point\": \"[0.0596, 0.0544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.6459]\", \"lift_point\": \"[0.0442, 0.6459]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Katy Perry video on YouTube\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0945, 0.5782]\", \"lift_point\": \"[0.0945, 0.5782]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9547]\", \"lift_point\": \"[0.0461, 0.9547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.0544]\", \"lift_point\": \"[0.0596, 0.0544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.6459]\", \"lift_point\": \"[0.0442, 0.6459]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Katy Perry video on YouTube\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0945, 0.5782]\", \"lift_point\": \"[0.0945, 0.5782]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.6407]\", \"lift_point\": \"[0.607, 0.6407]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.6315]\", \"lift_point\": \"[0.5618, 0.6315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6427, 0.8874]\", \"lift_point\": \"[0.6427, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9547]\", \"lift_point\": \"[0.0461, 0.9547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.0544]\", \"lift_point\": \"[0.0596, 0.0544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.6459]\", \"lift_point\": \"[0.0442, 0.6459]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Katy Perry video on YouTube\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0945, 0.5782]\", \"lift_point\": \"[0.0945, 0.5782]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.6407]\", \"lift_point\": \"[0.607, 0.6407]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8533_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6369]\", \"lift_point\": \"[0.7883, 0.6369]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8540_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6369]\", \"lift_point\": \"[0.7883, 0.6369]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2507, 0.3178]\", \"lift_point\": \"[0.2507, 0.3178]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8540_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6369]\", \"lift_point\": \"[0.7883, 0.6369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2507, 0.3178]\", \"lift_point\": \"[0.2507, 0.3178]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8540_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6369]\", \"lift_point\": \"[0.7883, 0.6369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2507, 0.3178]\", \"lift_point\": \"[0.2507, 0.3178]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit?\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1049, 0.418]\", \"lift_point\": \"[0.1049, 0.418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8540_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6369]\", \"lift_point\": \"[0.7883, 0.6369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2507, 0.3178]\", \"lift_point\": \"[0.2507, 0.3178]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1049, 0.418]\", \"lift_point\": \"[0.1049, 0.418]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3298, 0.457]\", \"lift_point\": \"[0.3298, 0.457]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8540_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6369]\", \"lift_point\": \"[0.7883, 0.6369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2507, 0.3178]\", \"lift_point\": \"[0.2507, 0.3178]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1049, 0.418]\", \"lift_point\": \"[0.1049, 0.418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3298, 0.457]\", \"lift_point\": \"[0.3298, 0.457]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4655, 0.7441]\", \"lift_point\": \"[0.4655, 0.7441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8540_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7883, 0.6369]\", \"lift_point\": \"[0.7883, 0.6369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2507, 0.3178]\", \"lift_point\": \"[0.2507, 0.3178]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1049, 0.418]\", \"lift_point\": \"[0.1049, 0.418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3298, 0.457]\", \"lift_point\": \"[0.3298, 0.457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4655, 0.7441]\", \"lift_point\": \"[0.4655, 0.7441]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8540_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8587_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8142, 0.616]\", \"lift_point\": \"[0.8142, 0.616]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8587_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8142, 0.616]\", \"lift_point\": \"[0.8142, 0.616]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.3675]\", \"lift_point\": \"[0.0612, 0.3675]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8587_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8142, 0.616]\", \"lift_point\": \"[0.8142, 0.616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.3675]\", \"lift_point\": \"[0.0612, 0.3675]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in planetary science?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8587_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8142, 0.616]\", \"lift_point\": \"[0.8142, 0.616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.3675]\", \"lift_point\": \"[0.0612, 0.3675]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in planetary science?\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in planetary science?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8587_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8142, 0.616]\", \"lift_point\": \"[0.8142, 0.616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.3675]\", \"lift_point\": \"[0.0612, 0.3675]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in planetary science?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in planetary science?\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1043, 0.6775]\", \"lift_point\": \"[0.1043, 0.6775]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8587_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8142, 0.616]\", \"lift_point\": \"[0.8142, 0.616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.3675]\", \"lift_point\": \"[0.0612, 0.3675]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in planetary science?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1043, 0.6775]\", \"lift_point\": \"[0.1043, 0.6775]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8587_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8248, 0.6278]\", \"lift_point\": \"[0.8248, 0.6278]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8248, 0.6278]\", \"lift_point\": \"[0.8248, 0.6278]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.5042]\", \"lift_point\": \"[0.0607, 0.5042]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8248, 0.6278]\", \"lift_point\": \"[0.8248, 0.6278]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.5042]\", \"lift_point\": \"[0.0607, 0.5042]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8248, 0.6278]\", \"lift_point\": \"[0.8248, 0.6278]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.5042]\", \"lift_point\": \"[0.0607, 0.5042]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.6857]\", \"lift_point\": \"[0.1281, 0.6857]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8248, 0.6278]\", \"lift_point\": \"[0.8248, 0.6278]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.5042]\", \"lift_point\": \"[0.0607, 0.5042]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.6857]\", \"lift_point\": \"[0.1281, 0.6857]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8248, 0.6278]\", \"lift_point\": \"[0.8248, 0.6278]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.5042]\", \"lift_point\": \"[0.0607, 0.5042]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.6857]\", \"lift_point\": \"[0.1281, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2906, 0.4312]\", \"lift_point\": \"[0.2906, 0.4312]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8248, 0.6278]\", \"lift_point\": \"[0.8248, 0.6278]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.5042]\", \"lift_point\": \"[0.0607, 0.5042]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.6857]\", \"lift_point\": \"[0.1281, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2906, 0.4312]\", \"lift_point\": \"[0.2906, 0.4312]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8617_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the News app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the News app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the News app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6819, 0.6296]\", \"lift_point\": \"[0.6819, 0.6296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6819, 0.6296]\", \"lift_point\": \"[0.6819, 0.6296]\", \"typed_text\": \"\" Goal: Install the News app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4424]\", \"lift_point\": \"[0.043, 0.4424]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6819, 0.6296]\", \"lift_point\": \"[0.6819, 0.6296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4424]\", \"lift_point\": \"[0.043, 0.4424]\", \"typed_text\": \"\" Goal: Install the News app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6819, 0.6296]\", \"lift_point\": \"[0.6819, 0.6296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4424]\", \"lift_point\": \"[0.043, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news app\" Goal: Install the News app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.102, 0.4834]\", \"lift_point\": \"[0.102, 0.4834]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6819, 0.6296]\", \"lift_point\": \"[0.6819, 0.6296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4424]\", \"lift_point\": \"[0.043, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.102, 0.4834]\", \"lift_point\": \"[0.102, 0.4834]\", \"typed_text\": \"\" Goal: Install the News app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4082, 0.2497]\", \"lift_point\": \"[0.4082, 0.2497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6819, 0.6296]\", \"lift_point\": \"[0.6819, 0.6296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4424]\", \"lift_point\": \"[0.043, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.102, 0.4834]\", \"lift_point\": \"[0.102, 0.4834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4082, 0.2497]\", \"lift_point\": \"[0.4082, 0.2497]\", \"typed_text\": \"\" Goal: Install the News app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3235, 0.383]\", \"lift_point\": \"[0.3235, 0.383]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6819, 0.6296]\", \"lift_point\": \"[0.6819, 0.6296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.043, 0.4424]\", \"lift_point\": \"[0.043, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.102, 0.4834]\", \"lift_point\": \"[0.102, 0.4834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4082, 0.2497]\", \"lift_point\": \"[0.4082, 0.2497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3235, 0.383]\", \"lift_point\": \"[0.3235, 0.383]\", \"typed_text\": \"\" Goal: Install the News app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8635_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5443, 0.8308]\", \"lift_point\": \"[0.5443, 0.8308]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5443, 0.8308]\", \"lift_point\": \"[0.5443, 0.8308]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.9343]\", \"lift_point\": \"[0.0465, 0.9343]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1577, 0.43]\", \"lift_point\": \"[0.1577, 0.43]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5443, 0.8308]\", \"lift_point\": \"[0.5443, 0.8308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.9343]\", \"lift_point\": \"[0.0465, 0.9343]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"atest news in planetary science?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.7894]\", \"lift_point\": \"[0.0644, 0.7894]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5443, 0.8308]\", \"lift_point\": \"[0.5443, 0.8308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.9343]\", \"lift_point\": \"[0.0465, 0.9343]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"atest news in planetary science?\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1111, 0.206]\", \"lift_point\": \"[0.1111, 0.206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5443, 0.8308]\", \"lift_point\": \"[0.5443, 0.8308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.9343]\", \"lift_point\": \"[0.0465, 0.9343]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"atest news in planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1111, 0.206]\", \"lift_point\": \"[0.1111, 0.206]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1704, 0.206]\", \"lift_point\": \"[0.1704, 0.206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4217]\", \"lift_point\": \"[0.8837, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5443, 0.8308]\", \"lift_point\": \"[0.5443, 0.8308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.9343]\", \"lift_point\": \"[0.0465, 0.9343]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"atest news in planetary science?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1111, 0.206]\", \"lift_point\": \"[0.1111, 0.206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1704, 0.206]\", \"lift_point\": \"[0.1704, 0.206]\", \"typed_text\": \"\" Goal: What's the latest news in planetary science?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8696_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6379, 0.8623]\", \"lift_point\": \"[0.6379, 0.8623]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6379, 0.8623]\", \"lift_point\": \"[0.6379, 0.8623]\", \"typed_text\": \"\" Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.0777]\", \"lift_point\": \"[0.0459, 0.0777]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6379, 0.8623]\", \"lift_point\": \"[0.6379, 0.8623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.0777]\", \"lift_point\": \"[0.0459, 0.0777]\", \"typed_text\": \"\" Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.6695]\", \"lift_point\": \"[0.0483, 0.6695]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6379, 0.8623]\", \"lift_point\": \"[0.6379, 0.8623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.0777]\", \"lift_point\": \"[0.0459, 0.0777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.6695]\", \"lift_point\": \"[0.0483, 0.6695]\", \"typed_text\": \"\" Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Maroon 5 video \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6379, 0.8623]\", \"lift_point\": \"[0.6379, 0.8623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.0777]\", \"lift_point\": \"[0.0459, 0.0777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.6695]\", \"lift_point\": \"[0.0483, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Maroon 5 video \" Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.4559]\", \"lift_point\": \"[0.1008, 0.4559]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6379, 0.8623]\", \"lift_point\": \"[0.6379, 0.8623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.0777]\", \"lift_point\": \"[0.0459, 0.0777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.6695]\", \"lift_point\": \"[0.0483, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Maroon 5 video \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.4559]\", \"lift_point\": \"[0.1008, 0.4559]\", \"typed_text\": \"\" Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5695, 0.7339]\", \"lift_point\": \"[0.5695, 0.7339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6379, 0.8623]\", \"lift_point\": \"[0.6379, 0.8623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.0777]\", \"lift_point\": \"[0.0459, 0.0777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.6695]\", \"lift_point\": \"[0.0483, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Maroon 5 video \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.4559]\", \"lift_point\": \"[0.1008, 0.4559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5695, 0.7339]\", \"lift_point\": \"[0.5695, 0.7339]\", \"typed_text\": \"\" Goal: Play the new Maroon 5 video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/8866_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend? \" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1266, 0.4876]\", \"lift_point\": \"[0.1266, 0.4876]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1266, 0.4876]\", \"lift_point\": \"[0.1266, 0.4876]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5402, 0.8672]\", \"lift_point\": \"[0.5402, 0.8672]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1266, 0.4876]\", \"lift_point\": \"[0.1266, 0.4876]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5402, 0.8672]\", \"lift_point\": \"[0.5402, 0.8672]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3666]\", \"lift_point\": \"[0.1171, 0.3666]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.6942]\", \"lift_point\": \"[0.4708, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1266, 0.4876]\", \"lift_point\": \"[0.1266, 0.4876]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5402, 0.8672]\", \"lift_point\": \"[0.5402, 0.8672]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3666]\", \"lift_point\": \"[0.1171, 0.3666]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2471, 0.3168]\", \"lift_point\": \"[0.2471, 0.3168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.4908]\", \"lift_point\": \"[0.116, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1266, 0.4876]\", \"lift_point\": \"[0.1266, 0.4876]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5402, 0.8672]\", \"lift_point\": \"[0.5402, 0.8672]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3666]\", \"lift_point\": \"[0.1171, 0.3666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2471, 0.3168]\", \"lift_point\": \"[0.2471, 0.3168]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8057, 0.9555]\", \"lift_point\": \"[0.8057, 0.9555]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9059]\", \"lift_point\": \"[0.066, 0.9059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.5821]\", \"lift_point\": \"[0.0412, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1266, 0.4876]\", \"lift_point\": \"[0.1266, 0.4876]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5402, 0.8672]\", \"lift_point\": \"[0.5402, 0.8672]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3666]\", \"lift_point\": \"[0.1171, 0.3666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2471, 0.3168]\", \"lift_point\": \"[0.2471, 0.3168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8057, 0.9555]\", \"lift_point\": \"[0.8057, 0.9555]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9357_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8443, 0.6292]\", \"lift_point\": \"[0.8443, 0.6292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9610_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8443, 0.6292]\", \"lift_point\": \"[0.8443, 0.6292]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.5916]\", \"lift_point\": \"[0.0563, 0.5916]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9610_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8443, 0.6292]\", \"lift_point\": \"[0.8443, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.5916]\", \"lift_point\": \"[0.0563, 0.5916]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in cosmology\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9610_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8443, 0.6292]\", \"lift_point\": \"[0.8443, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.5916]\", \"lift_point\": \"[0.0563, 0.5916]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in cosmology\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.3772]\", \"lift_point\": \"[0.1179, 0.3772]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9610_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8443, 0.6292]\", \"lift_point\": \"[0.8443, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.5916]\", \"lift_point\": \"[0.0563, 0.5916]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in cosmology\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.3772]\", \"lift_point\": \"[0.1179, 0.3772]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3328, 0.4183]\", \"lift_point\": \"[0.3328, 0.4183]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9610_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8443, 0.6292]\", \"lift_point\": \"[0.8443, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.5916]\", \"lift_point\": \"[0.0563, 0.5916]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in cosmology\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.3772]\", \"lift_point\": \"[0.1179, 0.3772]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3328, 0.4183]\", \"lift_point\": \"[0.3328, 0.4183]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.5215]\", \"lift_point\": \"[0.7234, 0.5215]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9610_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8443, 0.6292]\", \"lift_point\": \"[0.8443, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.5916]\", \"lift_point\": \"[0.0563, 0.5916]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in cosmology\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.3772]\", \"lift_point\": \"[0.1179, 0.3772]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3328, 0.4183]\", \"lift_point\": \"[0.3328, 0.4183]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.5215]\", \"lift_point\": \"[0.7234, 0.5215]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9610_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8233, 0.6003]\", \"lift_point\": \"[0.8233, 0.6003]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9777_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8233, 0.6003]\", \"lift_point\": \"[0.8233, 0.6003]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.834, 0.6061]\", \"lift_point\": \"[0.834, 0.6061]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9777_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8233, 0.6003]\", \"lift_point\": \"[0.8233, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.834, 0.6061]\", \"lift_point\": \"[0.834, 0.6061]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.467]\", \"lift_point\": \"[0.0592, 0.467]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9777_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8233, 0.6003]\", \"lift_point\": \"[0.8233, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.834, 0.6061]\", \"lift_point\": \"[0.834, 0.6061]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.467]\", \"lift_point\": \"[0.0592, 0.467]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Boston\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9777_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8233, 0.6003]\", \"lift_point\": \"[0.8233, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.834, 0.6061]\", \"lift_point\": \"[0.834, 0.6061]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.467]\", \"lift_point\": \"[0.0592, 0.467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Boston\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9777_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8233, 0.6003]\", \"lift_point\": \"[0.8233, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.834, 0.6061]\", \"lift_point\": \"[0.834, 0.6061]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.467]\", \"lift_point\": \"[0.0592, 0.467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Boston\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.2949]\", \"lift_point\": \"[0.3276, 0.2949]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9777_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8233, 0.6003]\", \"lift_point\": \"[0.8233, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.834, 0.6061]\", \"lift_point\": \"[0.834, 0.6061]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.467]\", \"lift_point\": \"[0.0592, 0.467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Boston\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.2949]\", \"lift_point\": \"[0.3276, 0.2949]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9777_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.6625]\", \"lift_point\": \"[0.0412, 0.6625]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.6625]\", \"lift_point\": \"[0.0412, 0.6625]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the 1000-Watt EGO Power+ Snow Blower\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.6625]\", \"lift_point\": \"[0.0412, 0.6625]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the 1000-Watt EGO Power+ Snow Blower\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.3049]\", \"lift_point\": \"[0.1092, 0.3049]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.6625]\", \"lift_point\": \"[0.0412, 0.6625]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the 1000-Watt EGO Power+ Snow Blower\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.3049]\", \"lift_point\": \"[0.1092, 0.3049]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.6625]\", \"lift_point\": \"[0.0412, 0.6625]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the 1000-Watt EGO Power+ Snow Blower\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.3049]\", \"lift_point\": \"[0.1092, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.6625]\", \"lift_point\": \"[0.0412, 0.6625]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the 1000-Watt EGO Power+ Snow Blower\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.3049]\", \"lift_point\": \"[0.1092, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3369, 0.4648]\", \"lift_point\": \"[0.3369, 0.4648]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8515, 0.6043]\", \"lift_point\": \"[0.8515, 0.6043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0412, 0.6625]\", \"lift_point\": \"[0.0412, 0.6625]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the 1000-Watt EGO Power+ Snow Blower\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.3049]\", \"lift_point\": \"[0.1092, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3369, 0.4648]\", \"lift_point\": \"[0.3369, 0.4648]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9794_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6878, 0.8905]\", \"lift_point\": \"[0.6878, 0.8905]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9931_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6878, 0.8905]\", \"lift_point\": \"[0.6878, 0.8905]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9931_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6878, 0.8905]\", \"lift_point\": \"[0.6878, 0.8905]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8466, 0.6103]\", \"lift_point\": \"[0.8466, 0.6103]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9931_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6878, 0.8905]\", \"lift_point\": \"[0.6878, 0.8905]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8466, 0.6103]\", \"lift_point\": \"[0.8466, 0.6103]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.6506]\", \"lift_point\": \"[0.0492, 0.6506]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9931_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6878, 0.8905]\", \"lift_point\": \"[0.6878, 0.8905]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8466, 0.6103]\", \"lift_point\": \"[0.8466, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.6506]\", \"lift_point\": \"[0.0492, 0.6506]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Lowes\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9931_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6878, 0.8905]\", \"lift_point\": \"[0.6878, 0.8905]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8466, 0.6103]\", \"lift_point\": \"[0.8466, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.6506]\", \"lift_point\": \"[0.0492, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Lowes\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9931_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6878, 0.8905]\", \"lift_point\": \"[0.6878, 0.8905]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8466, 0.6103]\", \"lift_point\": \"[0.8466, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.6506]\", \"lift_point\": \"[0.0492, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Lowes\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9931_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4329]\", \"lift_point\": \"[0.1363, 0.4329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4329]\", \"lift_point\": \"[0.1363, 0.4329]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1667, 0.1689]\", \"lift_point\": \"[0.1667, 0.1689]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4329]\", \"lift_point\": \"[0.1363, 0.4329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1667, 0.1689]\", \"lift_point\": \"[0.1667, 0.1689]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3062, 0.4479]\", \"lift_point\": \"[0.3062, 0.4479]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4329]\", \"lift_point\": \"[0.1363, 0.4329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1667, 0.1689]\", \"lift_point\": \"[0.1667, 0.1689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3062, 0.4479]\", \"lift_point\": \"[0.3062, 0.4479]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4329]\", \"lift_point\": \"[0.1363, 0.4329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1667, 0.1689]\", \"lift_point\": \"[0.1667, 0.1689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3062, 0.4479]\", \"lift_point\": \"[0.3062, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1958, 0.4127]\", \"lift_point\": \"[0.1958, 0.4127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4329]\", \"lift_point\": \"[0.1363, 0.4329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1667, 0.1689]\", \"lift_point\": \"[0.1667, 0.1689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3062, 0.4479]\", \"lift_point\": \"[0.3062, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1958, 0.4127]\", \"lift_point\": \"[0.1958, 0.4127]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3548, 0.1196]\", \"lift_point\": \"[0.3548, 0.1196]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.833, 0.606]\", \"lift_point\": \"[0.833, 0.606]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.8065]\", \"lift_point\": \"[0.0672, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1363, 0.4329]\", \"lift_point\": \"[0.1363, 0.4329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1667, 0.1689]\", \"lift_point\": \"[0.1667, 0.1689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3062, 0.4479]\", \"lift_point\": \"[0.3062, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1958, 0.4127]\", \"lift_point\": \"[0.1958, 0.4127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3548, 0.1196]\", \"lift_point\": \"[0.3548, 0.1196]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/9994_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9226, 0.4547]\", \"lift_point\": \"[0.9226, 0.4547]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10035_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9226, 0.4547]\", \"lift_point\": \"[0.9226, 0.4547]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10035_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9226, 0.4547]\", \"lift_point\": \"[0.9226, 0.4547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.4485]\", \"lift_point\": \"[0.1297, 0.4485]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10035_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9226, 0.4547]\", \"lift_point\": \"[0.9226, 0.4547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.4485]\", \"lift_point\": \"[0.1297, 0.4485]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10035_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9226, 0.4547]\", \"lift_point\": \"[0.9226, 0.4547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.4485]\", \"lift_point\": \"[0.1297, 0.4485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10035_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9226, 0.4547]\", \"lift_point\": \"[0.9226, 0.4547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.4485]\", \"lift_point\": \"[0.1297, 0.4485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6053]\", \"lift_point\": \"[0.4832, 0.6053]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10035_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9226, 0.4547]\", \"lift_point\": \"[0.9226, 0.4547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.4485]\", \"lift_point\": \"[0.1297, 0.4485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6053]\", \"lift_point\": \"[0.4832, 0.6053]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10035_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10060_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9194, 0.2965]\", \"lift_point\": \"[0.9194, 0.2965]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10060_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9194, 0.2965]\", \"lift_point\": \"[0.9194, 0.2965]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.3636]\", \"lift_point\": \"[0.0682, 0.3636]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10060_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9194, 0.2965]\", \"lift_point\": \"[0.9194, 0.2965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.3636]\", \"lift_point\": \"[0.0682, 0.3636]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10060_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9194, 0.2965]\", \"lift_point\": \"[0.9194, 0.2965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.3636]\", \"lift_point\": \"[0.0682, 0.3636]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.603]\", \"lift_point\": \"[0.1176, 0.603]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10060_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9194, 0.2965]\", \"lift_point\": \"[0.9194, 0.2965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.3636]\", \"lift_point\": \"[0.0682, 0.3636]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.603]\", \"lift_point\": \"[0.1176, 0.603]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10060_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9428, 0.524]\", \"lift_point\": \"[0.9428, 0.524]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10083_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9428, 0.524]\", \"lift_point\": \"[0.9428, 0.524]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.271]\", \"lift_point\": \"[0.0667, 0.271]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10083_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9428, 0.524]\", \"lift_point\": \"[0.9428, 0.524]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.271]\", \"lift_point\": \"[0.0667, 0.271]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10083_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9428, 0.524]\", \"lift_point\": \"[0.9428, 0.524]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.271]\", \"lift_point\": \"[0.0667, 0.271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland?\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2718, 0.5393]\", \"lift_point\": \"[0.2718, 0.5393]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10083_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9428, 0.524]\", \"lift_point\": \"[0.9428, 0.524]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.271]\", \"lift_point\": \"[0.0667, 0.271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2718, 0.5393]\", \"lift_point\": \"[0.2718, 0.5393]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10083_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9014, 0.4658]\", \"lift_point\": \"[0.9014, 0.4658]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10202_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9014, 0.4658]\", \"lift_point\": \"[0.9014, 0.4658]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.3868]\", \"lift_point\": \"[0.0603, 0.3868]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10202_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9014, 0.4658]\", \"lift_point\": \"[0.9014, 0.4658]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.3868]\", \"lift_point\": \"[0.0603, 0.3868]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Boston\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10202_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9014, 0.4658]\", \"lift_point\": \"[0.9014, 0.4658]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.3868]\", \"lift_point\": \"[0.0603, 0.3868]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Boston\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.272]\", \"lift_point\": \"[0.1319, 0.272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10202_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9014, 0.4658]\", \"lift_point\": \"[0.9014, 0.4658]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.3868]\", \"lift_point\": \"[0.0603, 0.3868]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.272]\", \"lift_point\": \"[0.1319, 0.272]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10202_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9014, 0.4658]\", \"lift_point\": \"[0.9014, 0.4658]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.3868]\", \"lift_point\": \"[0.0603, 0.3868]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.272]\", \"lift_point\": \"[0.1319, 0.272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.3859]\", \"lift_point\": \"[0.5594, 0.3859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10202_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9014, 0.4658]\", \"lift_point\": \"[0.9014, 0.4658]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.3868]\", \"lift_point\": \"[0.0603, 0.3868]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.272]\", \"lift_point\": \"[0.1319, 0.272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.3859]\", \"lift_point\": \"[0.5594, 0.3859]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10202_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.1523]\", \"lift_point\": \"[0.607, 0.1523]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.1523]\", \"lift_point\": \"[0.607, 0.1523]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2242, 0.3622]\", \"lift_point\": \"[0.2242, 0.3622]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.1523]\", \"lift_point\": \"[0.607, 0.1523]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2242, 0.3622]\", \"lift_point\": \"[0.2242, 0.3622]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.3798]\", \"lift_point\": \"[0.0716, 0.3798]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.1523]\", \"lift_point\": \"[0.607, 0.1523]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2242, 0.3622]\", \"lift_point\": \"[0.2242, 0.3622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.3798]\", \"lift_point\": \"[0.0716, 0.3798]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.1523]\", \"lift_point\": \"[0.607, 0.1523]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2242, 0.3622]\", \"lift_point\": \"[0.2242, 0.3622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.3798]\", \"lift_point\": \"[0.0716, 0.3798]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4034, 0.4373]\", \"lift_point\": \"[0.4034, 0.4373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0914, 0.497]\", \"lift_point\": \"[0.0914, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.607, 0.1523]\", \"lift_point\": \"[0.607, 0.1523]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2242, 0.3622]\", \"lift_point\": \"[0.2242, 0.3622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0716, 0.3798]\", \"lift_point\": \"[0.0716, 0.3798]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4034, 0.4373]\", \"lift_point\": \"[0.4034, 0.4373]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10211_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.2988]\", \"lift_point\": \"[0.0693, 0.2988]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.2988]\", \"lift_point\": \"[0.0693, 0.2988]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0822, 0.5471]\", \"lift_point\": \"[0.0822, 0.5471]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.2988]\", \"lift_point\": \"[0.0693, 0.2988]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0822, 0.5471]\", \"lift_point\": \"[0.0822, 0.5471]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.2988]\", \"lift_point\": \"[0.0693, 0.2988]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0822, 0.5471]\", \"lift_point\": \"[0.0822, 0.5471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6315, 0.6047]\", \"lift_point\": \"[0.6315, 0.6047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.2988]\", \"lift_point\": \"[0.0693, 0.2988]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0822, 0.5471]\", \"lift_point\": \"[0.0822, 0.5471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6315, 0.6047]\", \"lift_point\": \"[0.6315, 0.6047]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1344, 0.2843]\", \"lift_point\": \"[0.1344, 0.2843]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.2988]\", \"lift_point\": \"[0.0693, 0.2988]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0822, 0.5471]\", \"lift_point\": \"[0.0822, 0.5471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6315, 0.6047]\", \"lift_point\": \"[0.6315, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1344, 0.2843]\", \"lift_point\": \"[0.1344, 0.2843]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1405, 0.4311]\", \"lift_point\": \"[0.1405, 0.4311]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8139, 0.8516]\", \"lift_point\": \"[0.8139, 0.8516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0693, 0.2988]\", \"lift_point\": \"[0.0693, 0.2988]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0822, 0.5471]\", \"lift_point\": \"[0.0822, 0.5471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6315, 0.6047]\", \"lift_point\": \"[0.6315, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1344, 0.2843]\", \"lift_point\": \"[0.1344, 0.2843]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1405, 0.4311]\", \"lift_point\": \"[0.1405, 0.4311]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10257_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.4142]\", \"lift_point\": \"[0.9271, 0.4142]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10275_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.4142]\", \"lift_point\": \"[0.9271, 0.4142]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.3245]\", \"lift_point\": \"[0.0575, 0.3245]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10275_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.4142]\", \"lift_point\": \"[0.9271, 0.4142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.3245]\", \"lift_point\": \"[0.0575, 0.3245]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.4669]\", \"lift_point\": \"[0.0672, 0.4669]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10275_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.4142]\", \"lift_point\": \"[0.9271, 0.4142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.3245]\", \"lift_point\": \"[0.0575, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.4669]\", \"lift_point\": \"[0.0672, 0.4669]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10275_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.4142]\", \"lift_point\": \"[0.9271, 0.4142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.3245]\", \"lift_point\": \"[0.0575, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.4669]\", \"lift_point\": \"[0.0672, 0.4669]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Paris\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1173, 0.4034]\", \"lift_point\": \"[0.1173, 0.4034]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10275_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.4142]\", \"lift_point\": \"[0.9271, 0.4142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.3245]\", \"lift_point\": \"[0.0575, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.4669]\", \"lift_point\": \"[0.0672, 0.4669]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1173, 0.4034]\", \"lift_point\": \"[0.1173, 0.4034]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10275_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10320_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5299, 0.1896]\", \"lift_point\": \"[0.5299, 0.1896]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10320_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5299, 0.1896]\", \"lift_point\": \"[0.5299, 0.1896]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10320_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6254]\", \"lift_point\": \"[0.8001, 0.6254]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10481_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6254]\", \"lift_point\": \"[0.8001, 0.6254]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.9678]\", \"lift_point\": \"[0.0557, 0.9678]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10481_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6254]\", \"lift_point\": \"[0.8001, 0.6254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.9678]\", \"lift_point\": \"[0.0557, 0.9678]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.9775]\", \"lift_point\": \"[0.0581, 0.9775]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10481_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6254]\", \"lift_point\": \"[0.8001, 0.6254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.9678]\", \"lift_point\": \"[0.0557, 0.9678]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.9775]\", \"lift_point\": \"[0.0581, 0.9775]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10481_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [TYPE,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6141]\", \"lift_point\": \"[0.799, 0.6141]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6141]\", \"lift_point\": \"[0.799, 0.6141]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.8498]\", \"lift_point\": \"[0.5409, 0.8498]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6141]\", \"lift_point\": \"[0.799, 0.6141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.8498]\", \"lift_point\": \"[0.5409, 0.8498]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.643]\", \"lift_point\": \"[0.0483, 0.643]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6141]\", \"lift_point\": \"[0.799, 0.6141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.8498]\", \"lift_point\": \"[0.5409, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.643]\", \"lift_point\": \"[0.0483, 0.643]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.6213]\", \"lift_point\": \"[0.359, 0.6213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6141]\", \"lift_point\": \"[0.799, 0.6141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.8498]\", \"lift_point\": \"[0.5409, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.643]\", \"lift_point\": \"[0.0483, 0.643]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.4027]\", \"lift_point\": \"[0.1231, 0.4027]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.5729]\", \"lift_point\": \"[0.0614, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6141]\", \"lift_point\": \"[0.799, 0.6141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.8498]\", \"lift_point\": \"[0.5409, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.643]\", \"lift_point\": \"[0.0483, 0.643]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.4027]\", \"lift_point\": \"[0.1231, 0.4027]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6198, 0.4425]\", \"lift_point\": \"[0.6198, 0.4425]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do i get to the nearest Home Depot?\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6141]\", \"lift_point\": \"[0.799, 0.6141]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5409, 0.8498]\", \"lift_point\": \"[0.5409, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.643]\", \"lift_point\": \"[0.0483, 0.643]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.4027]\", \"lift_point\": \"[0.1231, 0.4027]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6198, 0.4425]\", \"lift_point\": \"[0.6198, 0.4425]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10532_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Calendar app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.6226]\", \"lift_point\": \"[0.0578, 0.6226]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.6226]\", \"lift_point\": \"[0.0578, 0.6226]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Calendar\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.6226]\", \"lift_point\": \"[0.0578, 0.6226]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Calendar\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2094]\", \"lift_point\": \"[0.1135, 0.2094]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.6226]\", \"lift_point\": \"[0.0578, 0.6226]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Calendar\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2094]\", \"lift_point\": \"[0.1135, 0.2094]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4035, 0.3219]\", \"lift_point\": \"[0.4035, 0.3219]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.6226]\", \"lift_point\": \"[0.0578, 0.6226]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Calendar\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2094]\", \"lift_point\": \"[0.1135, 0.2094]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4035, 0.3219]\", \"lift_point\": \"[0.4035, 0.3219]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2111, 0.2851]\", \"lift_point\": \"[0.2111, 0.2851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1739, 0.1727]\", \"lift_point\": \"[0.1739, 0.1727]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.6226]\", \"lift_point\": \"[0.0578, 0.6226]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Calendar\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2094]\", \"lift_point\": \"[0.1135, 0.2094]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4035, 0.3219]\", \"lift_point\": \"[0.4035, 0.3219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2111, 0.2851]\", \"lift_point\": \"[0.2111, 0.2851]\", \"typed_text\": \"\" Goal: Install the Calendar app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10691_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10816_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10816_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10816_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10816_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8629]\", \"lift_point\": \"[0.6423, 0.8629]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10816_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8629]\", \"lift_point\": \"[0.6423, 0.8629]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.889]\", \"lift_point\": \"[0.6476, 0.889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10816_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6644, 0.873]\", \"lift_point\": \"[0.6644, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6423, 0.8629]\", \"lift_point\": \"[0.6423, 0.8629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.889]\", \"lift_point\": \"[0.6476, 0.889]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10816_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10838_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.3583]\", \"lift_point\": \"[0.881, 0.3583]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10838_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.3583]\", \"lift_point\": \"[0.881, 0.3583]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Brazilian Real\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10838_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.3583]\", \"lift_point\": \"[0.881, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Brazilian Real\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10838_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.3583]\", \"lift_point\": \"[0.881, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Brazilian Real\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10838_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8885, 0.5074]\", \"lift_point\": \"[0.8885, 0.5074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8885, 0.5074]\", \"lift_point\": \"[0.8885, 0.5074]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8885, 0.5074]\", \"lift_point\": \"[0.8885, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8885, 0.5074]\", \"lift_point\": \"[0.8885, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8885, 0.5074]\", \"lift_point\": \"[0.8885, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.583, 0.5171]\", \"lift_point\": \"[0.583, 0.5171]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8885, 0.5074]\", \"lift_point\": \"[0.8885, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.583, 0.5171]\", \"lift_point\": \"[0.583, 0.5171]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7293, 0.7629]\", \"lift_point\": \"[0.7293, 0.7629]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8885, 0.5074]\", \"lift_point\": \"[0.8885, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.583, 0.5171]\", \"lift_point\": \"[0.583, 0.5171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7293, 0.7629]\", \"lift_point\": \"[0.7293, 0.7629]\", \"typed_text\": \"\" Goal: What's the top post on reddit?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10873_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Cheesecake Factory\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Cheesecake Factory\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Cheesecake Factory\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Cheesecake Factory\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Cheesecake Factory\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5323, 0.4879]\", \"lift_point\": \"[0.5323, 0.4879]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8821, 0.4104]\", \"lift_point\": \"[0.8821, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Cheesecake Factory\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5323, 0.4879]\", \"lift_point\": \"[0.5323, 0.4879]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/10901_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11038_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7766, 0.6942]\", \"lift_point\": \"[0.7766, 0.6942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11038_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7766, 0.6942]\", \"lift_point\": \"[0.7766, 0.6942]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.4273]\", \"lift_point\": \"[0.0437, 0.4273]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11038_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7766, 0.6942]\", \"lift_point\": \"[0.7766, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.4273]\", \"lift_point\": \"[0.0437, 0.4273]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11038_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7766, 0.6942]\", \"lift_point\": \"[0.7766, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.4273]\", \"lift_point\": \"[0.0437, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden?\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1335, 0.2836]\", \"lift_point\": \"[0.1335, 0.2836]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11038_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7766, 0.6942]\", \"lift_point\": \"[0.7766, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.4273]\", \"lift_point\": \"[0.0437, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1335, 0.2836]\", \"lift_point\": \"[0.1335, 0.2836]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11038_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Red Lobster?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11110_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\" Goal: What's on the menu at Red Lobster?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5517]\", \"lift_point\": \"[0.048, 0.5517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11110_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5517]\", \"lift_point\": \"[0.048, 0.5517]\", \"typed_text\": \"\" Goal: What's on the menu at Red Lobster?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at red lobster?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11110_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5517]\", \"lift_point\": \"[0.048, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at red lobster?\" Goal: What's on the menu at Red Lobster?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.249]\", \"lift_point\": \"[0.1333, 0.249]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11110_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5517]\", \"lift_point\": \"[0.048, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at red lobster?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.249]\", \"lift_point\": \"[0.1333, 0.249]\", \"typed_text\": \"\" Goal: What's on the menu at Red Lobster?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11110_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5517]\", \"lift_point\": \"[0.048, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at red lobster?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.249]\", \"lift_point\": \"[0.1333, 0.249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Red Lobster?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5788, 0.2193]\", \"lift_point\": \"[0.5788, 0.2193]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11110_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7834, 0.698]\", \"lift_point\": \"[0.7834, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.5517]\", \"lift_point\": \"[0.048, 0.5517]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at red lobster?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.249]\", \"lift_point\": \"[0.1333, 0.249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5788, 0.2193]\", \"lift_point\": \"[0.5788, 0.2193]\", \"typed_text\": \"\" Goal: What's on the menu at Red Lobster?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11110_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11124_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.499]\", \"lift_point\": \"[0.3718, 0.499]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11124_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.499]\", \"lift_point\": \"[0.3718, 0.499]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4043]\", \"lift_point\": \"[0.0464, 0.4043]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11124_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.499]\", \"lift_point\": \"[0.3718, 0.499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4043]\", \"lift_point\": \"[0.0464, 0.4043]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11124_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.499]\", \"lift_point\": \"[0.3718, 0.499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4043]\", \"lift_point\": \"[0.0464, 0.4043]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.2131]\", \"lift_point\": \"[0.1985, 0.2131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11124_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.499]\", \"lift_point\": \"[0.3718, 0.499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4043]\", \"lift_point\": \"[0.0464, 0.4043]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.2131]\", \"lift_point\": \"[0.1985, 0.2131]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4812, 0.2229]\", \"lift_point\": \"[0.4812, 0.2229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11124_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.499]\", \"lift_point\": \"[0.3718, 0.499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4043]\", \"lift_point\": \"[0.0464, 0.4043]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.2131]\", \"lift_point\": \"[0.1985, 0.2131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4812, 0.2229]\", \"lift_point\": \"[0.4812, 0.2229]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11124_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Argentina>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.6632]\", \"lift_point\": \"[0.7736, 0.6632]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11152_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.6632]\", \"lift_point\": \"[0.7736, 0.6632]\", \"typed_text\": \"\" Goal: Google the capital of Argentina>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0424, 0.461]\", \"lift_point\": \"[0.0424, 0.461]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11152_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.6632]\", \"lift_point\": \"[0.7736, 0.6632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0424, 0.461]\", \"lift_point\": \"[0.0424, 0.461]\", \"typed_text\": \"\" Goal: Google the capital of Argentina>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Argentina\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11152_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.6632]\", \"lift_point\": \"[0.7736, 0.6632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0424, 0.461]\", \"lift_point\": \"[0.0424, 0.461]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Argentina\" Goal: Google the capital of Argentina>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1417, 0.3636]\", \"lift_point\": \"[0.1417, 0.3636]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11152_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.6632]\", \"lift_point\": \"[0.7736, 0.6632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0424, 0.461]\", \"lift_point\": \"[0.0424, 0.461]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Argentina\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1417, 0.3636]\", \"lift_point\": \"[0.1417, 0.3636]\", \"typed_text\": \"\" Goal: Google the capital of Argentina>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11152_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach bag?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6827]\", \"lift_point\": \"[0.7825, 0.6827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11296_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6827]\", \"lift_point\": \"[0.7825, 0.6827]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach bag?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.4489]\", \"lift_point\": \"[0.0504, 0.4489]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11296_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6827]\", \"lift_point\": \"[0.7825, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.4489]\", \"lift_point\": \"[0.0504, 0.4489]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach bag?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach bag?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11296_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6827]\", \"lift_point\": \"[0.7825, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.4489]\", \"lift_point\": \"[0.0504, 0.4489]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach bag?\" Goal: Where can I buy a nice beach bag?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.2757]\", \"lift_point\": \"[0.1411, 0.2757]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11296_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6827]\", \"lift_point\": \"[0.7825, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.4489]\", \"lift_point\": \"[0.0504, 0.4489]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach bag?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.2757]\", \"lift_point\": \"[0.1411, 0.2757]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach bag?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11296_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6827]\", \"lift_point\": \"[0.7825, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.4489]\", \"lift_point\": \"[0.0504, 0.4489]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach bag?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.2757]\", \"lift_point\": \"[0.1411, 0.2757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach bag?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.2143]\", \"lift_point\": \"[0.1899, 0.2143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11296_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6827]\", \"lift_point\": \"[0.7825, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.4489]\", \"lift_point\": \"[0.0504, 0.4489]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach bag?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.2757]\", \"lift_point\": \"[0.1411, 0.2757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.2143]\", \"lift_point\": \"[0.1899, 0.2143]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach bag?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11296_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.6763]\", \"lift_point\": \"[0.7836, 0.6763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.6763]\", \"lift_point\": \"[0.7836, 0.6763]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.2705]\", \"lift_point\": \"[0.066, 0.2705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.6763]\", \"lift_point\": \"[0.7836, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.2705]\", \"lift_point\": \"[0.066, 0.2705]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach blanket?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.6763]\", \"lift_point\": \"[0.7836, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.2705]\", \"lift_point\": \"[0.066, 0.2705]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach blanket?\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.235]\", \"lift_point\": \"[0.1411, 0.235]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.6763]\", \"lift_point\": \"[0.7836, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.2705]\", \"lift_point\": \"[0.066, 0.2705]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach blanket?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.235]\", \"lift_point\": \"[0.1411, 0.235]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.6763]\", \"lift_point\": \"[0.7836, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.2705]\", \"lift_point\": \"[0.066, 0.2705]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach blanket?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.235]\", \"lift_point\": \"[0.1411, 0.235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.2896]\", \"lift_point\": \"[0.4708, 0.2896]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.6763]\", \"lift_point\": \"[0.7836, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.2705]\", \"lift_point\": \"[0.066, 0.2705]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach blanket?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.235]\", \"lift_point\": \"[0.1411, 0.235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4708, 0.2896]\", \"lift_point\": \"[0.4708, 0.2896]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach blanket?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11384_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.678]\", \"lift_point\": \"[0.7804, 0.678]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.678]\", \"lift_point\": \"[0.7804, 0.678]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0421, 0.3922]\", \"lift_point\": \"[0.0421, 0.3922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.678]\", \"lift_point\": \"[0.7804, 0.678]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0421, 0.3922]\", \"lift_point\": \"[0.0421, 0.3922]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.678]\", \"lift_point\": \"[0.7804, 0.678]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0421, 0.3922]\", \"lift_point\": \"[0.0421, 0.3922]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1269, 0.3198]\", \"lift_point\": \"[0.1269, 0.3198]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.678]\", \"lift_point\": \"[0.7804, 0.678]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0421, 0.3922]\", \"lift_point\": \"[0.0421, 0.3922]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1269, 0.3198]\", \"lift_point\": \"[0.1269, 0.3198]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.678]\", \"lift_point\": \"[0.7804, 0.678]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0421, 0.3922]\", \"lift_point\": \"[0.0421, 0.3922]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1269, 0.3198]\", \"lift_point\": \"[0.1269, 0.3198]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2736, 0.269]\", \"lift_point\": \"[0.2736, 0.269]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.678]\", \"lift_point\": \"[0.7804, 0.678]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0421, 0.3922]\", \"lift_point\": \"[0.0421, 0.3922]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1269, 0.3198]\", \"lift_point\": \"[0.1269, 0.3198]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2736, 0.269]\", \"lift_point\": \"[0.2736, 0.269]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11392_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a lion?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6884]\", \"lift_point\": \"[0.788, 0.6884]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11407_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6884]\", \"lift_point\": \"[0.788, 0.6884]\", \"typed_text\": \"\" Goal: How big is a lion?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0413, 0.4771]\", \"lift_point\": \"[0.0413, 0.4771]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11407_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6884]\", \"lift_point\": \"[0.788, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0413, 0.4771]\", \"lift_point\": \"[0.0413, 0.4771]\", \"typed_text\": \"\" Goal: How big is a lion?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"big is a lion?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11407_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6884]\", \"lift_point\": \"[0.788, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0413, 0.4771]\", \"lift_point\": \"[0.0413, 0.4771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"big is a lion?\" Goal: How big is a lion?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.2413]\", \"lift_point\": \"[0.1387, 0.2413]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11407_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6884]\", \"lift_point\": \"[0.788, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0413, 0.4771]\", \"lift_point\": \"[0.0413, 0.4771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"big is a lion?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.2413]\", \"lift_point\": \"[0.1387, 0.2413]\", \"typed_text\": \"\" Goal: How big is a lion?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8387, 0.5262]\", \"lift_point\": \"[0.8387, 0.5262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11407_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6884]\", \"lift_point\": \"[0.788, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0413, 0.4771]\", \"lift_point\": \"[0.0413, 0.4771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"big is a lion?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.2413]\", \"lift_point\": \"[0.1387, 0.2413]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8387, 0.5262]\", \"lift_point\": \"[0.8387, 0.5262]\", \"typed_text\": \"\" Goal: How big is a lion?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11407_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6301, 0.1398]\", \"lift_point\": \"[0.6301, 0.1398]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6301, 0.1398]\", \"lift_point\": \"[0.6301, 0.1398]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6301, 0.1398]\", \"lift_point\": \"[0.6301, 0.1398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7723, 0.6778]\", \"lift_point\": \"[0.7723, 0.6778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6301, 0.1398]\", \"lift_point\": \"[0.6301, 0.1398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.1467]\", \"lift_point\": \"[0.1898, 0.1467]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.3652]\", \"lift_point\": \"[0.0473, 0.3652]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.3202]\", \"lift_point\": \"[0.136, 0.3202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6301, 0.1398]\", \"lift_point\": \"[0.6301, 0.1398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.1467]\", \"lift_point\": \"[0.1898, 0.1467]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11465_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calculator>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4403]\", \"lift_point\": \"[0.0685, 0.4403]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4403]\", \"lift_point\": \"[0.0685, 0.4403]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calculator \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4403]\", \"lift_point\": \"[0.0685, 0.4403]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calculator \" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.4406]\", \"lift_point\": \"[0.1461, 0.4406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4403]\", \"lift_point\": \"[0.0685, 0.4403]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calculator \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.4406]\", \"lift_point\": \"[0.1461, 0.4406]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.3562]\", \"lift_point\": \"[0.3076, 0.3562]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4403]\", \"lift_point\": \"[0.0685, 0.4403]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calculator \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.4406]\", \"lift_point\": \"[0.1461, 0.4406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.3562]\", \"lift_point\": \"[0.3076, 0.3562]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.7567]\", \"lift_point\": \"[0.6333, 0.7567]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.1221]\", \"lift_point\": \"[0.6324, 0.1221]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4403]\", \"lift_point\": \"[0.0685, 0.4403]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calculator \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.4406]\", \"lift_point\": \"[0.1461, 0.4406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.3562]\", \"lift_point\": \"[0.3076, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.7567]\", \"lift_point\": \"[0.6333, 0.7567]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5973, 0.8227]\", \"lift_point\": \"[0.5973, 0.8227]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0594, 0.4623]\", \"lift_point\": \"[0.0594, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.9216]\", \"lift_point\": \"[0.0583, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4403]\", \"lift_point\": \"[0.0685, 0.4403]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calculator \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.4406]\", \"lift_point\": \"[0.1461, 0.4406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.3562]\", \"lift_point\": \"[0.3076, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.7567]\", \"lift_point\": \"[0.6333, 0.7567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5973, 0.8227]\", \"lift_point\": \"[0.5973, 0.8227]\", \"typed_text\": \"\" Goal: Open the calculator>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11730_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11768_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.5887]\", \"lift_point\": \"[0.6481, 0.5887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11768_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.5887]\", \"lift_point\": \"[0.6481, 0.5887]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3639]\", \"lift_point\": \"[0.0529, 0.3639]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11768_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.5887]\", \"lift_point\": \"[0.6481, 0.5887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3639]\", \"lift_point\": \"[0.0529, 0.3639]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" burger restaurants \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11768_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.5887]\", \"lift_point\": \"[0.6481, 0.5887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3639]\", \"lift_point\": \"[0.0529, 0.3639]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" burger restaurants \" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1507, 0.369]\", \"lift_point\": \"[0.1507, 0.369]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11768_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.5887]\", \"lift_point\": \"[0.6481, 0.5887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3639]\", \"lift_point\": \"[0.0529, 0.3639]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" burger restaurants \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1507, 0.369]\", \"lift_point\": \"[0.1507, 0.369]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11768_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5943, 0.3915]\", \"lift_point\": \"[0.5943, 0.3915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.3807]\", \"lift_point\": \"[0.3179, 0.3807]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2742, 0.3428]\", \"lift_point\": \"[0.2742, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.3807]\", \"lift_point\": \"[0.3179, 0.3807]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.0913]\", \"lift_point\": \"[0.0521, 0.0913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9113]\", \"lift_point\": \"[0.0705, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.3807]\", \"lift_point\": \"[0.3179, 0.3807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.0913]\", \"lift_point\": \"[0.0521, 0.0913]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.2362]\", \"lift_point\": \"[0.4937, 0.2362]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.4191]\", \"lift_point\": \"[0.0633, 0.4191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.3807]\", \"lift_point\": \"[0.3179, 0.3807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.0913]\", \"lift_point\": \"[0.0521, 0.0913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.2362]\", \"lift_point\": \"[0.4937, 0.2362]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5199, 0.2492]\", \"lift_point\": \"[0.5199, 0.2492]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the calendar and show me this weeks events? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.3807]\", \"lift_point\": \"[0.3179, 0.3807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.0913]\", \"lift_point\": \"[0.0521, 0.0913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.2362]\", \"lift_point\": \"[0.4937, 0.2362]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5199, 0.2492]\", \"lift_point\": \"[0.5199, 0.2492]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2941, 0.9229]\", \"lift_point\": \"[0.2941, 0.9229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.4739]\", \"lift_point\": \"[0.1329, 0.4739]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.3807]\", \"lift_point\": \"[0.3179, 0.3807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.0913]\", \"lift_point\": \"[0.0521, 0.0913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4937, 0.2362]\", \"lift_point\": \"[0.4937, 0.2362]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5199, 0.2492]\", \"lift_point\": \"[0.5199, 0.2492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2941, 0.9229]\", \"lift_point\": \"[0.2941, 0.9229]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11946_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5979, 0.3934]\", \"lift_point\": \"[0.5979, 0.3934]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5979, 0.3934]\", \"lift_point\": \"[0.5979, 0.3934]\", \"typed_text\": \"\" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.6564]\", \"lift_point\": \"[0.1291, 0.6564]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5979, 0.3934]\", \"lift_point\": \"[0.5979, 0.3934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.6564]\", \"lift_point\": \"[0.1291, 0.6564]\", \"typed_text\": \"\" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9036]\", \"lift_point\": \"[0.056, 0.9036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5979, 0.3934]\", \"lift_point\": \"[0.5979, 0.3934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.6564]\", \"lift_point\": \"[0.1291, 0.6564]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9036]\", \"lift_point\": \"[0.056, 0.9036]\", \"typed_text\": \"\" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4158]\", \"lift_point\": \"[0.0562, 0.4158]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5979, 0.3934]\", \"lift_point\": \"[0.5979, 0.3934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.6564]\", \"lift_point\": \"[0.1291, 0.6564]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9036]\", \"lift_point\": \"[0.056, 0.9036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4158]\", \"lift_point\": \"[0.0562, 0.4158]\", \"typed_text\": \"\" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the BBC \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5979, 0.3934]\", \"lift_point\": \"[0.5979, 0.3934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.6564]\", \"lift_point\": \"[0.1291, 0.6564]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9036]\", \"lift_point\": \"[0.056, 0.9036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4158]\", \"lift_point\": \"[0.0562, 0.4158]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the BBC \" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.4168]\", \"lift_point\": \"[0.1322, 0.4168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5979, 0.3934]\", \"lift_point\": \"[0.5979, 0.3934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.6564]\", \"lift_point\": \"[0.1291, 0.6564]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9036]\", \"lift_point\": \"[0.056, 0.9036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4158]\", \"lift_point\": \"[0.0562, 0.4158]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the BBC \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.4168]\", \"lift_point\": \"[0.1322, 0.4168]\", \"typed_text\": \"\" Goal: Play the latest video from the BBC>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11978_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.9235]\", \"lift_point\": \"[0.0694, 0.9235]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.9235]\", \"lift_point\": \"[0.0694, 0.9235]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.4433]\", \"lift_point\": \"[0.0602, 0.4433]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.9235]\", \"lift_point\": \"[0.0694, 0.9235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.4433]\", \"lift_point\": \"[0.0602, 0.4433]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Boston \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.2609]\", \"lift_point\": \"[0.131, 0.2609]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.9235]\", \"lift_point\": \"[0.0694, 0.9235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.4433]\", \"lift_point\": \"[0.0602, 0.4433]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Boston \" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2667, 0.3762]\", \"lift_point\": \"[0.2667, 0.3762]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.9235]\", \"lift_point\": \"[0.0694, 0.9235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.4433]\", \"lift_point\": \"[0.0602, 0.4433]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Boston \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2667, 0.3762]\", \"lift_point\": \"[0.2667, 0.3762]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3474, 0.272]\", \"lift_point\": \"[0.3474, 0.272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.9235]\", \"lift_point\": \"[0.0694, 0.9235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.4433]\", \"lift_point\": \"[0.0602, 0.4433]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Boston \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2667, 0.3762]\", \"lift_point\": \"[0.2667, 0.3762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3474, 0.272]\", \"lift_point\": \"[0.3474, 0.272]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7326, 0.5677]\", \"lift_point\": \"[0.7326, 0.5677]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6091, 0.3766]\", \"lift_point\": \"[0.6091, 0.3766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2833, 0.5076]\", \"lift_point\": \"[0.2833, 0.5076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.9235]\", \"lift_point\": \"[0.0694, 0.9235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.4433]\", \"lift_point\": \"[0.0602, 0.4433]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Boston \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2667, 0.3762]\", \"lift_point\": \"[0.2667, 0.3762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3474, 0.272]\", \"lift_point\": \"[0.3474, 0.272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7326, 0.5677]\", \"lift_point\": \"[0.7326, 0.5677]\", \"typed_text\": \"\" Goal: Search for hotels in Boston>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/11991_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2795, 0.5272]\", \"lift_point\": \"[0.2795, 0.5272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2795, 0.5272]\", \"lift_point\": \"[0.2795, 0.5272]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8952]\", \"lift_point\": \"[0.0638, 0.8952]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2795, 0.5272]\", \"lift_point\": \"[0.2795, 0.5272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8952]\", \"lift_point\": \"[0.0638, 0.8952]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.442]\", \"lift_point\": \"[0.0585, 0.442]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2795, 0.5272]\", \"lift_point\": \"[0.2795, 0.5272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8952]\", \"lift_point\": \"[0.0638, 0.8952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.442]\", \"lift_point\": \"[0.0585, 0.442]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2795, 0.5272]\", \"lift_point\": \"[0.2795, 0.5272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8952]\", \"lift_point\": \"[0.0638, 0.8952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.442]\", \"lift_point\": \"[0.0585, 0.442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology? \" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1497, 0.5314]\", \"lift_point\": \"[0.1497, 0.5314]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2795, 0.5272]\", \"lift_point\": \"[0.2795, 0.5272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8952]\", \"lift_point\": \"[0.0638, 0.8952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.442]\", \"lift_point\": \"[0.0585, 0.442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1497, 0.5314]\", \"lift_point\": \"[0.1497, 0.5314]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.4227]\", \"lift_point\": \"[0.3273, 0.4227]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6142, 0.3825]\", \"lift_point\": \"[0.6142, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2795, 0.5272]\", \"lift_point\": \"[0.2795, 0.5272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.8952]\", \"lift_point\": \"[0.0638, 0.8952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.442]\", \"lift_point\": \"[0.0585, 0.442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1497, 0.5314]\", \"lift_point\": \"[0.1497, 0.5314]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.4227]\", \"lift_point\": \"[0.3273, 0.4227]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12082_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2881, 0.6019]\", \"lift_point\": \"[0.2881, 0.6019]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2881, 0.6019]\", \"lift_point\": \"[0.2881, 0.6019]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.9207]\", \"lift_point\": \"[0.0669, 0.9207]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2881, 0.6019]\", \"lift_point\": \"[0.2881, 0.6019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.9207]\", \"lift_point\": \"[0.0669, 0.9207]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3997]\", \"lift_point\": \"[0.0677, 0.3997]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2881, 0.6019]\", \"lift_point\": \"[0.2881, 0.6019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.9207]\", \"lift_point\": \"[0.0669, 0.9207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3997]\", \"lift_point\": \"[0.0677, 0.3997]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2881, 0.6019]\", \"lift_point\": \"[0.2881, 0.6019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.9207]\", \"lift_point\": \"[0.0669, 0.9207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3997]\", \"lift_point\": \"[0.0677, 0.3997]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2881, 0.6019]\", \"lift_point\": \"[0.2881, 0.6019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.9207]\", \"lift_point\": \"[0.0669, 0.9207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3997]\", \"lift_point\": \"[0.0677, 0.3997]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store? \" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.4251]\", \"lift_point\": \"[0.1231, 0.4251]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5977, 0.366]\", \"lift_point\": \"[0.5977, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2881, 0.6019]\", \"lift_point\": \"[0.2881, 0.6019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0669, 0.9207]\", \"lift_point\": \"[0.0669, 0.9207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3997]\", \"lift_point\": \"[0.0677, 0.3997]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.4251]\", \"lift_point\": \"[0.1231, 0.4251]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12131_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Malaysia? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Malaysia? \" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1342, 0.3641]\", \"lift_point\": \"[0.1342, 0.3641]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Malaysia? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1342, 0.3641]\", \"lift_point\": \"[0.1342, 0.3641]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.3769]\", \"lift_point\": \"[0.3034, 0.3769]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Malaysia? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1342, 0.3641]\", \"lift_point\": \"[0.1342, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.3769]\", \"lift_point\": \"[0.3034, 0.3769]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5905, 0.3914]\", \"lift_point\": \"[0.5905, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Malaysia? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1342, 0.3641]\", \"lift_point\": \"[0.1342, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.3769]\", \"lift_point\": \"[0.3034, 0.3769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8207, 0.56]\", \"lift_point\": \"[0.8207, 0.56]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.5216]\", \"lift_point\": \"[0.28, 0.5216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.8865]\", \"lift_point\": \"[0.0525, 0.8865]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.418]\", \"lift_point\": \"[0.0505, 0.418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Malaysia? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1342, 0.3641]\", \"lift_point\": \"[0.1342, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.3769]\", \"lift_point\": \"[0.3034, 0.3769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8207, 0.56]\", \"lift_point\": \"[0.8207, 0.56]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12382_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.9124]\", \"lift_point\": \"[0.0719, 0.9124]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.9124]\", \"lift_point\": \"[0.0719, 0.9124]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0466, 0.5782]\", \"lift_point\": \"[0.0466, 0.5782]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2999, 0.2785]\", \"lift_point\": \"[0.2999, 0.2785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.9124]\", \"lift_point\": \"[0.0719, 0.9124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0466, 0.5782]\", \"lift_point\": \"[0.0466, 0.5782]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.9124]\", \"lift_point\": \"[0.0719, 0.9124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0466, 0.5782]\", \"lift_point\": \"[0.0466, 0.5782]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology? \" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.202, 0.5932]\", \"lift_point\": \"[0.202, 0.5932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.595, 0.3585]\", \"lift_point\": \"[0.595, 0.3585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.9124]\", \"lift_point\": \"[0.0719, 0.9124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0466, 0.5782]\", \"lift_point\": \"[0.0466, 0.5782]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.202, 0.5932]\", \"lift_point\": \"[0.202, 0.5932]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3258, 0.5227]\", \"lift_point\": \"[0.3258, 0.5227]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5935, 0.3583]\", \"lift_point\": \"[0.5935, 0.3583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2931, 0.4892]\", \"lift_point\": \"[0.2931, 0.4892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.9124]\", \"lift_point\": \"[0.0719, 0.9124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0466, 0.5782]\", \"lift_point\": \"[0.0466, 0.5782]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.202, 0.5932]\", \"lift_point\": \"[0.202, 0.5932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3258, 0.5227]\", \"lift_point\": \"[0.3258, 0.5227]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12411_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8023, 0.6164]\", \"lift_point\": \"[0.8023, 0.6164]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12597_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8023, 0.6164]\", \"lift_point\": \"[0.8023, 0.6164]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5783]\", \"lift_point\": \"[0.0661, 0.5783]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12597_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8023, 0.6164]\", \"lift_point\": \"[0.8023, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5783]\", \"lift_point\": \"[0.0661, 0.5783]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12597_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8023, 0.6164]\", \"lift_point\": \"[0.8023, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5783]\", \"lift_point\": \"[0.0661, 0.5783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time?\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1276, 0.3746]\", \"lift_point\": \"[0.1276, 0.3746]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12597_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8023, 0.6164]\", \"lift_point\": \"[0.8023, 0.6164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5783]\", \"lift_point\": \"[0.0661, 0.5783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1276, 0.3746]\", \"lift_point\": \"[0.1276, 0.3746]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12597_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.6256]\", \"lift_point\": \"[0.804, 0.6256]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12639_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.6256]\", \"lift_point\": \"[0.804, 0.6256]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5621]\", \"lift_point\": \"[0.0651, 0.5621]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12639_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.6256]\", \"lift_point\": \"[0.804, 0.6256]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5621]\", \"lift_point\": \"[0.0651, 0.5621]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach chair?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12639_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.6256]\", \"lift_point\": \"[0.804, 0.6256]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5621]\", \"lift_point\": \"[0.0651, 0.5621]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach chair?\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.5464]\", \"lift_point\": \"[0.1322, 0.5464]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12639_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.6256]\", \"lift_point\": \"[0.804, 0.6256]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5621]\", \"lift_point\": \"[0.0651, 0.5621]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach chair?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.5464]\", \"lift_point\": \"[0.1322, 0.5464]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12639_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.6256]\", \"lift_point\": \"[0.804, 0.6256]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5621]\", \"lift_point\": \"[0.0651, 0.5621]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach chair?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.5464]\", \"lift_point\": \"[0.1322, 0.5464]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7367, 0.4438]\", \"lift_point\": \"[0.7367, 0.4438]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12639_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.6256]\", \"lift_point\": \"[0.804, 0.6256]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5621]\", \"lift_point\": \"[0.0651, 0.5621]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach chair?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.5464]\", \"lift_point\": \"[0.1322, 0.5464]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7367, 0.4438]\", \"lift_point\": \"[0.7367, 0.4438]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12639_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.4827]\", \"lift_point\": \"[0.129, 0.4827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.4827]\", \"lift_point\": \"[0.129, 0.4827]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.4827]\", \"lift_point\": \"[0.129, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.4827]\", \"lift_point\": \"[0.129, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.4827]\", \"lift_point\": \"[0.129, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8082, 0.6143]\", \"lift_point\": \"[0.8082, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.4827]\", \"lift_point\": \"[0.129, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8991, 0.4974]\", \"lift_point\": \"[0.8991, 0.4974]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.5173]\", \"lift_point\": \"[0.0743, 0.5173]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.129, 0.4827]\", \"lift_point\": \"[0.129, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8991, 0.4974]\", \"lift_point\": \"[0.8991, 0.4974]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12684_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8086, 0.6235]\", \"lift_point\": \"[0.8086, 0.6235]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12745_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8086, 0.6235]\", \"lift_point\": \"[0.8086, 0.6235]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4897]\", \"lift_point\": \"[0.0652, 0.4897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12745_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8086, 0.6235]\", \"lift_point\": \"[0.8086, 0.6235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4897]\", \"lift_point\": \"[0.0652, 0.4897]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12745_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8086, 0.6235]\", \"lift_point\": \"[0.8086, 0.6235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4897]\", \"lift_point\": \"[0.0652, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.4961]\", \"lift_point\": \"[0.131, 0.4961]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12745_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8086, 0.6235]\", \"lift_point\": \"[0.8086, 0.6235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4897]\", \"lift_point\": \"[0.0652, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.131, 0.4961]\", \"lift_point\": \"[0.131, 0.4961]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12745_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6236]\", \"lift_point\": \"[0.8079, 0.6236]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12758_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6236]\", \"lift_point\": \"[0.8079, 0.6236]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.5484]\", \"lift_point\": \"[0.0697, 0.5484]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12758_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6236]\", \"lift_point\": \"[0.8079, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.5484]\", \"lift_point\": \"[0.0697, 0.5484]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Panama\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12758_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6236]\", \"lift_point\": \"[0.8079, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.5484]\", \"lift_point\": \"[0.0697, 0.5484]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Panama\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.3266]\", \"lift_point\": \"[0.1297, 0.3266]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12758_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6236]\", \"lift_point\": \"[0.8079, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.5484]\", \"lift_point\": \"[0.0697, 0.5484]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Panama\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.3266]\", \"lift_point\": \"[0.1297, 0.3266]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12758_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8104, 0.6183]\", \"lift_point\": \"[0.8104, 0.6183]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12786_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8104, 0.6183]\", \"lift_point\": \"[0.8104, 0.6183]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.566]\", \"lift_point\": \"[0.0584, 0.566]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12786_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8104, 0.6183]\", \"lift_point\": \"[0.8104, 0.6183]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.566]\", \"lift_point\": \"[0.0584, 0.566]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12786_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8104, 0.6183]\", \"lift_point\": \"[0.8104, 0.6183]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.566]\", \"lift_point\": \"[0.0584, 0.566]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3566]\", \"lift_point\": \"[0.134, 0.3566]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12786_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8104, 0.6183]\", \"lift_point\": \"[0.8104, 0.6183]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.566]\", \"lift_point\": \"[0.0584, 0.566]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3566]\", \"lift_point\": \"[0.134, 0.3566]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12786_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8051, 0.6184]\", \"lift_point\": \"[0.8051, 0.6184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12802_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8051, 0.6184]\", \"lift_point\": \"[0.8051, 0.6184]\", \"typed_text\": \"\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5295]\", \"lift_point\": \"[0.0534, 0.5295]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12802_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8051, 0.6184]\", \"lift_point\": \"[0.8051, 0.6184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5295]\", \"lift_point\": \"[0.0534, 0.5295]\", \"typed_text\": \"\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Subway?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12802_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8051, 0.6184]\", \"lift_point\": \"[0.8051, 0.6184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5295]\", \"lift_point\": \"[0.0534, 0.5295]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Subway?\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3645]\", \"lift_point\": \"[0.1332, 0.3645]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12802_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8051, 0.6184]\", \"lift_point\": \"[0.8051, 0.6184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5295]\", \"lift_point\": \"[0.0534, 0.5295]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Subway?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3645]\", \"lift_point\": \"[0.1332, 0.3645]\", \"typed_text\": \"\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12802_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Mexico city to Sydney>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12815_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Sydney>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8071, 0.6207]\", \"lift_point\": \"[0.8071, 0.6207]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12815_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8071, 0.6207]\", \"lift_point\": \"[0.8071, 0.6207]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Sydney>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.472]\", \"lift_point\": \"[0.0554, 0.472]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12815_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8071, 0.6207]\", \"lift_point\": \"[0.8071, 0.6207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.472]\", \"lift_point\": \"[0.0554, 0.472]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Sydney>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Sydney\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12815_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8071, 0.6207]\", \"lift_point\": \"[0.8071, 0.6207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.472]\", \"lift_point\": \"[0.0554, 0.472]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Sydney\" Goal: Search for flights from Mexico city to Sydney>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3904]\", \"lift_point\": \"[0.1329, 0.3904]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12815_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8071, 0.6207]\", \"lift_point\": \"[0.8071, 0.6207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.472]\", \"lift_point\": \"[0.0554, 0.472]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3904]\", \"lift_point\": \"[0.1329, 0.3904]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Sydney>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12815_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.4655]\", \"lift_point\": \"[0.0565, 0.4655]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.4655]\", \"lift_point\": \"[0.0565, 0.4655]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.4655]\", \"lift_point\": \"[0.0565, 0.4655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.4084]\", \"lift_point\": \"[0.1257, 0.4084]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.4655]\", \"lift_point\": \"[0.0565, 0.4655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.4084]\", \"lift_point\": \"[0.1257, 0.4084]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.4655]\", \"lift_point\": \"[0.0565, 0.4655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.4084]\", \"lift_point\": \"[0.1257, 0.4084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.4655]\", \"lift_point\": \"[0.0565, 0.4655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.4084]\", \"lift_point\": \"[0.1257, 0.4084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.666, 0.2675]\", \"lift_point\": \"[0.666, 0.2675]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8138, 0.6186]\", \"lift_point\": \"[0.8138, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.4655]\", \"lift_point\": \"[0.0565, 0.4655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.4084]\", \"lift_point\": \"[0.1257, 0.4084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.666, 0.2675]\", \"lift_point\": \"[0.666, 0.2675]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12837_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12986_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8078, 0.6252]\", \"lift_point\": \"[0.8078, 0.6252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12986_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8078, 0.6252]\", \"lift_point\": \"[0.8078, 0.6252]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.479]\", \"lift_point\": \"[0.0561, 0.479]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12986_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8078, 0.6252]\", \"lift_point\": \"[0.8078, 0.6252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.479]\", \"lift_point\": \"[0.0561, 0.479]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Time in San Francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12986_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8078, 0.6252]\", \"lift_point\": \"[0.8078, 0.6252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.479]\", \"lift_point\": \"[0.0561, 0.479]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Time in San Francisco\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.3725]\", \"lift_point\": \"[0.1292, 0.3725]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12986_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8078, 0.6252]\", \"lift_point\": \"[0.8078, 0.6252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.479]\", \"lift_point\": \"[0.0561, 0.479]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Time in San Francisco\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.3725]\", \"lift_point\": \"[0.1292, 0.3725]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/12986_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8089, 0.6237]\", \"lift_point\": \"[0.8089, 0.6237]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13003_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8089, 0.6237]\", \"lift_point\": \"[0.8089, 0.6237]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3469]\", \"lift_point\": \"[0.0529, 0.3469]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13003_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8089, 0.6237]\", \"lift_point\": \"[0.8089, 0.6237]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3469]\", \"lift_point\": \"[0.0529, 0.3469]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13003_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8089, 0.6237]\", \"lift_point\": \"[0.8089, 0.6237]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3469]\", \"lift_point\": \"[0.0529, 0.3469]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1372, 0.4749]\", \"lift_point\": \"[0.1372, 0.4749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13003_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8089, 0.6237]\", \"lift_point\": \"[0.8089, 0.6237]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.3469]\", \"lift_point\": \"[0.0529, 0.3469]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1372, 0.4749]\", \"lift_point\": \"[0.1372, 0.4749]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13003_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.8382]\", \"lift_point\": \"[0.1643, 0.8382]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.8382]\", \"lift_point\": \"[0.1643, 0.8382]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.7749]\", \"lift_point\": \"[0.0628, 0.7749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.8382]\", \"lift_point\": \"[0.1643, 0.8382]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.7749]\", \"lift_point\": \"[0.0628, 0.7749]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.9298]\", \"lift_point\": \"[0.0609, 0.9298]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.8382]\", \"lift_point\": \"[0.1643, 0.8382]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.7749]\", \"lift_point\": \"[0.0628, 0.7749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.9298]\", \"lift_point\": \"[0.0609, 0.9298]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.8382]\", \"lift_point\": \"[0.1643, 0.8382]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.7749]\", \"lift_point\": \"[0.0628, 0.7749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.9298]\", \"lift_point\": \"[0.0609, 0.9298]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit app\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1361, 0.2898]\", \"lift_point\": \"[0.1361, 0.2898]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.8382]\", \"lift_point\": \"[0.1643, 0.8382]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.7749]\", \"lift_point\": \"[0.0628, 0.7749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.9298]\", \"lift_point\": \"[0.0609, 0.9298]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1361, 0.2898]\", \"lift_point\": \"[0.1361, 0.2898]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1505, 0.8499]\", \"lift_point\": \"[0.1505, 0.8499]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1643, 0.8382]\", \"lift_point\": \"[0.1643, 0.8382]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.7749]\", \"lift_point\": \"[0.0628, 0.7749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.9298]\", \"lift_point\": \"[0.0609, 0.9298]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1361, 0.2898]\", \"lift_point\": \"[0.1361, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1505, 0.8499]\", \"lift_point\": \"[0.1505, 0.8499]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13075_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7486, 0.2964]\", \"lift_point\": \"[0.7486, 0.2964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13147_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7486, 0.2964]\", \"lift_point\": \"[0.7486, 0.2964]\", \"typed_text\": \"\" Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0834, 0.841]\", \"lift_point\": \"[0.0834, 0.841]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13147_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7486, 0.2964]\", \"lift_point\": \"[0.7486, 0.2964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0834, 0.841]\", \"lift_point\": \"[0.0834, 0.841]\", \"typed_text\": \"\" Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.1098]\", \"lift_point\": \"[0.0692, 0.1098]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13147_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7486, 0.2964]\", \"lift_point\": \"[0.7486, 0.2964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0834, 0.841]\", \"lift_point\": \"[0.0834, 0.841]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.1098]\", \"lift_point\": \"[0.0692, 0.1098]\", \"typed_text\": \"\" Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13147_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.625, 0.6017]\", \"lift_point\": \"[0.625, 0.6017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.625, 0.6017]\", \"lift_point\": \"[0.625, 0.6017]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.4092]\", \"lift_point\": \"[0.1318, 0.4092]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.625, 0.6017]\", \"lift_point\": \"[0.625, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.4092]\", \"lift_point\": \"[0.1318, 0.4092]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.8927]\", \"lift_point\": \"[0.1332, 0.8927]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.625, 0.6017]\", \"lift_point\": \"[0.625, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.4092]\", \"lift_point\": \"[0.1318, 0.4092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.8927]\", \"lift_point\": \"[0.1332, 0.8927]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.625, 0.6017]\", \"lift_point\": \"[0.625, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.4092]\", \"lift_point\": \"[0.1318, 0.4092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.8927]\", \"lift_point\": \"[0.1332, 0.8927]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in paris\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.26, 0.3267]\", \"lift_point\": \"[0.26, 0.3267]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.625, 0.6017]\", \"lift_point\": \"[0.625, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.4092]\", \"lift_point\": \"[0.1318, 0.4092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.8927]\", \"lift_point\": \"[0.1332, 0.8927]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.26, 0.3267]\", \"lift_point\": \"[0.26, 0.3267]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2692, 0.4655]\", \"lift_point\": \"[0.2692, 0.4655]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.625, 0.6017]\", \"lift_point\": \"[0.625, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.4092]\", \"lift_point\": \"[0.1318, 0.4092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.8927]\", \"lift_point\": \"[0.1332, 0.8927]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.26, 0.3267]\", \"lift_point\": \"[0.26, 0.3267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2692, 0.4655]\", \"lift_point\": \"[0.2692, 0.4655]\", \"typed_text\": \"\" Goal: What's the weather like in Paris?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13349_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barcelona to Boston flight\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13370_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barcelona to Boston flight\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13370_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barcelona to Boston flight\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3091, 0.4261]\", \"lift_point\": \"[0.3091, 0.4261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13370_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barcelona to Boston flight\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3091, 0.4261]\", \"lift_point\": \"[0.3091, 0.4261]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.384, 0.3307]\", \"lift_point\": \"[0.384, 0.3307]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13370_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barcelona to Boston flight\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3091, 0.4261]\", \"lift_point\": \"[0.3091, 0.4261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.384, 0.3307]\", \"lift_point\": \"[0.384, 0.3307]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Boston>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13370_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.6137]\", \"lift_point\": \"[0.2387, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2501, 0.6092]\", \"lift_point\": \"[0.2501, 0.6092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2249, 0.6282]\", \"lift_point\": \"[0.2249, 0.6282]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1306, 0.3968]\", \"lift_point\": \"[0.1306, 0.3968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.8972]\", \"lift_point\": \"[0.1211, 0.8972]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5293]\", \"lift_point\": \"[0.2757, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4969, 0.411]\", \"lift_point\": \"[0.4969, 0.411]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4969, 0.411]\", \"lift_point\": \"[0.4969, 0.411]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4969, 0.411]\", \"lift_point\": \"[0.4969, 0.411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4969, 0.411]\", \"lift_point\": \"[0.4969, 0.411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4969, 0.411]\", \"lift_point\": \"[0.4969, 0.411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5772, 0.6842]\", \"lift_point\": \"[0.5772, 0.6842]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4969, 0.411]\", \"lift_point\": \"[0.4969, 0.411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5772, 0.6842]\", \"lift_point\": \"[0.5772, 0.6842]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13587_20.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.0365]\", \"lift_point\": \"[0.0575, 0.0365]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8912, 0.667]\", \"lift_point\": \"[0.8912, 0.667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8921, 0.3941]\", \"lift_point\": \"[0.8921, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8787, 0.3913]\", \"lift_point\": \"[0.8787, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7438, 0.7177]\", \"lift_point\": \"[0.7438, 0.7177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6992]\", \"lift_point\": \"[0.7785, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8149, 0.6966]\", \"lift_point\": \"[0.8149, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4065]\", \"lift_point\": \"[0.0722, 0.4065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.6482]\", \"lift_point\": \"[0.1351, 0.6482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2715, 0.6476]\", \"lift_point\": \"[0.2715, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0528, 0.2859]\", \"lift_point\": \"[0.0528, 0.2859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.4412]\", \"lift_point\": \"[0.8191, 0.4412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4471, 0.1787]\", \"lift_point\": \"[0.4471, 0.1787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.3148]\", \"lift_point\": \"[0.8042, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.5777]\", \"lift_point\": \"[0.6879, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2862]\", \"lift_point\": \"[0.8361, 0.2862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_54.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_55.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_56.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8366, 0.182]\", \"lift_point\": \"[0.8366, 0.182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_57.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.3885]\", \"lift_point\": \"[0.6461, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_58.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_59.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_60.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_61.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_62.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_63.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_64.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_65.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_66.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_67.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_68.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_69.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_70.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_71.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_72.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_73.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_74.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4645, 0.4475]\", \"lift_point\": \"[0.4645, 0.4475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_75.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_76.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_77.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_78.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_79.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_80.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_81.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_82.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.7333]\", \"lift_point\": \"[0.9, 0.7333]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_83.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_84.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_85.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_86.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_87.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_88.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_89.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_90.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_91.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4602, 0.5652]\", \"lift_point\": \"[0.4602, 0.5652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_92.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3384, 0.4751]\", \"lift_point\": \"[0.3384, 0.4751]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_93.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3384, 0.4751]\", \"lift_point\": \"[0.3384, 0.4751]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/13941_94.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.5777]\", \"lift_point\": \"[0.8064, 0.5777]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.5777]\", \"lift_point\": \"[0.8064, 0.5777]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.6632]\", \"lift_point\": \"[0.8087, 0.6632]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.5777]\", \"lift_point\": \"[0.8064, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.6632]\", \"lift_point\": \"[0.8087, 0.6632]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.9311]\", \"lift_point\": \"[0.0627, 0.9311]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.5777]\", \"lift_point\": \"[0.8064, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.6632]\", \"lift_point\": \"[0.8087, 0.6632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.9311]\", \"lift_point\": \"[0.0627, 0.9311]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.5777]\", \"lift_point\": \"[0.8064, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.6632]\", \"lift_point\": \"[0.8087, 0.6632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.9311]\", \"lift_point\": \"[0.0627, 0.9311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.5777]\", \"lift_point\": \"[0.8064, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.6632]\", \"lift_point\": \"[0.8087, 0.6632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.9311]\", \"lift_point\": \"[0.0627, 0.9311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.5777]\", \"lift_point\": \"[0.8064, 0.5777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.6632]\", \"lift_point\": \"[0.8087, 0.6632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.9311]\", \"lift_point\": \"[0.0627, 0.9311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14144_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find a good burger place on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7882, 0.6536]\", \"lift_point\": \"[0.7882, 0.6536]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14180_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7882, 0.6536]\", \"lift_point\": \"[0.7882, 0.6536]\", \"typed_text\": \"\" Goal: Find a good burger place on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1926, 0.7614]\", \"lift_point\": \"[0.1926, 0.7614]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14180_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7882, 0.6536]\", \"lift_point\": \"[0.7882, 0.6536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1926, 0.7614]\", \"lift_point\": \"[0.1926, 0.7614]\", \"typed_text\": \"\" Goal: Find a good burger place on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.9382]\", \"lift_point\": \"[0.1172, 0.9382]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14180_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7882, 0.6536]\", \"lift_point\": \"[0.7882, 0.6536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1926, 0.7614]\", \"lift_point\": \"[0.1926, 0.7614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.9382]\", \"lift_point\": \"[0.1172, 0.9382]\", \"typed_text\": \"\" Goal: Find a good burger place on Maps>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find a good burger place on Maps\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14180_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7882, 0.6536]\", \"lift_point\": \"[0.7882, 0.6536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1926, 0.7614]\", \"lift_point\": \"[0.1926, 0.7614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.9382]\", \"lift_point\": \"[0.1172, 0.9382]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find a good burger place on Maps\" Goal: Find a good burger place on Maps>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14180_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7882, 0.6536]\", \"lift_point\": \"[0.7882, 0.6536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1926, 0.7614]\", \"lift_point\": \"[0.1926, 0.7614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.9382]\", \"lift_point\": \"[0.1172, 0.9382]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find a good burger place on Maps\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Find a good burger place on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14180_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14231_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14231_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14231_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.5639]\", \"lift_point\": \"[0.0611, 0.5639]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14231_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.5639]\", \"lift_point\": \"[0.0611, 0.5639]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14231_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.5639]\", \"lift_point\": \"[0.0611, 0.5639]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14231_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.7148]\", \"lift_point\": \"[0.7915, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.5639]\", \"lift_point\": \"[0.0611, 0.5639]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14231_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.6885]\", \"lift_point\": \"[0.0496, 0.6885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.6885]\", \"lift_point\": \"[0.0496, 0.6885]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.6885]\", \"lift_point\": \"[0.0496, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7252]\", \"lift_point\": \"[0.7989, 0.7252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.6885]\", \"lift_point\": \"[0.0496, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"open\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.6885]\", \"lift_point\": \"[0.0496, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"open\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.682]\", \"lift_point\": \"[0.7928, 0.682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.6885]\", \"lift_point\": \"[0.0496, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"open\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3275, 0.6995]\", \"lift_point\": \"[0.3275, 0.6995]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6979]\", \"lift_point\": \"[0.7936, 0.6979]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.6885]\", \"lift_point\": \"[0.0496, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"open\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3275, 0.6995]\", \"lift_point\": \"[0.3275, 0.6995]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14304_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.679]\", \"lift_point\": \"[0.7989, 0.679]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14351_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.679]\", \"lift_point\": \"[0.7989, 0.679]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1811, 0.916]\", \"lift_point\": \"[0.1811, 0.916]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14351_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.679]\", \"lift_point\": \"[0.7989, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1811, 0.916]\", \"lift_point\": \"[0.1811, 0.916]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in venezuela\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14351_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.679]\", \"lift_point\": \"[0.7989, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1811, 0.916]\", \"lift_point\": \"[0.1811, 0.916]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in venezuela\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14351_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.679]\", \"lift_point\": \"[0.7989, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1811, 0.916]\", \"lift_point\": \"[0.1811, 0.916]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in venezuela\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2405, 0.7142]\", \"lift_point\": \"[0.2405, 0.7142]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14351_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.679]\", \"lift_point\": \"[0.7989, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1811, 0.916]\", \"lift_point\": \"[0.1811, 0.916]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in venezuela\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2405, 0.7142]\", \"lift_point\": \"[0.2405, 0.7142]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14351_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1808, 0.7277]\", \"lift_point\": \"[0.1808, 0.7277]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1808, 0.7277]\", \"lift_point\": \"[0.1808, 0.7277]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.9302]\", \"lift_point\": \"[0.1166, 0.9302]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1808, 0.7277]\", \"lift_point\": \"[0.1808, 0.7277]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.9302]\", \"lift_point\": \"[0.1166, 0.9302]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9645]\", \"lift_point\": \"[0.9857, 0.9645]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1808, 0.7277]\", \"lift_point\": \"[0.1808, 0.7277]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.9302]\", \"lift_point\": \"[0.1166, 0.9302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9645]\", \"lift_point\": \"[0.9857, 0.9645]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1808, 0.7277]\", \"lift_point\": \"[0.1808, 0.7277]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.9302]\", \"lift_point\": \"[0.1166, 0.9302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9645]\", \"lift_point\": \"[0.9857, 0.9645]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9815]\", \"lift_point\": \"[0.9857, 0.9815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1808, 0.7277]\", \"lift_point\": \"[0.1808, 0.7277]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.9302]\", \"lift_point\": \"[0.1166, 0.9302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9645]\", \"lift_point\": \"[0.9857, 0.9645]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9815]\", \"lift_point\": \"[0.9857, 0.9815]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6749]\", \"lift_point\": \"[0.7936, 0.6749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1808, 0.7277]\", \"lift_point\": \"[0.1808, 0.7277]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.9302]\", \"lift_point\": \"[0.1166, 0.9302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9645]\", \"lift_point\": \"[0.9857, 0.9645]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9857, 0.9815]\", \"lift_point\": \"[0.9857, 0.9815]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14379_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3708, 0.5372]\", \"lift_point\": \"[0.3708, 0.5372]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.035, 0.6353]\", \"lift_point\": \"[0.035, 0.6353]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.035, 0.6353]\", \"lift_point\": \"[0.035, 0.6353]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.035, 0.6353]\", \"lift_point\": \"[0.035, 0.6353]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"k?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.035, 0.6353]\", \"lift_point\": \"[0.035, 0.6353]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"k?\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.7034]\", \"lift_point\": \"[0.799, 0.7034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7124]\", \"lift_point\": \"[0.7989, 0.7124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.7363]\", \"lift_point\": \"[0.7885, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.035, 0.6353]\", \"lift_point\": \"[0.035, 0.6353]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"k?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14429_16.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.5726]\", \"lift_point\": \"[0.1952, 0.5726]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.5726]\", \"lift_point\": \"[0.1952, 0.5726]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.9375]\", \"lift_point\": \"[0.1147, 0.9375]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.5726]\", \"lift_point\": \"[0.1952, 0.5726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.9375]\", \"lift_point\": \"[0.1147, 0.9375]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Brazilian Real?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.5726]\", \"lift_point\": \"[0.1952, 0.5726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.9375]\", \"lift_point\": \"[0.1147, 0.9375]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Brazilian Real?\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6887]\", \"lift_point\": \"[0.7944, 0.6887]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.5726]\", \"lift_point\": \"[0.1952, 0.5726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.9375]\", \"lift_point\": \"[0.1147, 0.9375]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Brazilian Real?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Brazilian Real?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14463_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14602_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14602_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3734, 0.535]\", \"lift_point\": \"[0.3734, 0.535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14602_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3734, 0.535]\", \"lift_point\": \"[0.3734, 0.535]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1882, 0.9111]\", \"lift_point\": \"[0.1882, 0.9111]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14602_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3734, 0.535]\", \"lift_point\": \"[0.3734, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1882, 0.9111]\", \"lift_point\": \"[0.1882, 0.9111]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.0614]\", \"lift_point\": \"[0.1172, 0.0614]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14602_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3734, 0.535]\", \"lift_point\": \"[0.3734, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1882, 0.9111]\", \"lift_point\": \"[0.1882, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.0614]\", \"lift_point\": \"[0.1172, 0.0614]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1268, 0.4538]\", \"lift_point\": \"[0.1268, 0.4538]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14602_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3734, 0.535]\", \"lift_point\": \"[0.3734, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1882, 0.9111]\", \"lift_point\": \"[0.1882, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.0614]\", \"lift_point\": \"[0.1172, 0.0614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1268, 0.4538]\", \"lift_point\": \"[0.1268, 0.4538]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14602_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach tent?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach tent?\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2819]\", \"lift_point\": \"[0.1135, 0.2819]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach tent?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2819]\", \"lift_point\": \"[0.1135, 0.2819]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach tent?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2819]\", \"lift_point\": \"[0.1135, 0.2819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach tent?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2819]\", \"lift_point\": \"[0.1135, 0.2819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach tent?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2819]\", \"lift_point\": \"[0.1135, 0.2819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1022, 0.1564]\", \"lift_point\": \"[0.1022, 0.1564]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8412, 0.63]\", \"lift_point\": \"[0.8412, 0.63]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.5007]\", \"lift_point\": \"[0.0452, 0.5007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach tent?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1135, 0.2819]\", \"lift_point\": \"[0.1135, 0.2819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1022, 0.1564]\", \"lift_point\": \"[0.1022, 0.1564]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14647_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2357, 0.7324]\", \"lift_point\": \"[0.2357, 0.7324]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14676_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2357, 0.7324]\", \"lift_point\": \"[0.2357, 0.7324]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14676_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2357, 0.7324]\", \"lift_point\": \"[0.2357, 0.7324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7987, 0.6175]\", \"lift_point\": \"[0.7987, 0.6175]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14676_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2357, 0.7324]\", \"lift_point\": \"[0.2357, 0.7324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7987, 0.6175]\", \"lift_point\": \"[0.7987, 0.6175]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14676_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14769_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.613]\", \"lift_point\": \"[0.8356, 0.613]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14769_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.613]\", \"lift_point\": \"[0.8356, 0.613]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0386, 0.4054]\", \"lift_point\": \"[0.0386, 0.4054]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14769_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.613]\", \"lift_point\": \"[0.8356, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0386, 0.4054]\", \"lift_point\": \"[0.0386, 0.4054]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in San Francisco?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14769_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.613]\", \"lift_point\": \"[0.8356, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0386, 0.4054]\", \"lift_point\": \"[0.0386, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in San Francisco?\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.305]\", \"lift_point\": \"[0.1147, 0.305]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14769_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.613]\", \"lift_point\": \"[0.8356, 0.613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0386, 0.4054]\", \"lift_point\": \"[0.0386, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in San Francisco?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.305]\", \"lift_point\": \"[0.1147, 0.305]\", \"typed_text\": \"\" Goal: What's the weather like in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14769_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8401, 0.6155]\", \"lift_point\": \"[0.8401, 0.6155]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14844_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8401, 0.6155]\", \"lift_point\": \"[0.8401, 0.6155]\", \"typed_text\": \"\" Goal: What's a good restaurant in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0338, 0.4432]\", \"lift_point\": \"[0.0338, 0.4432]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14844_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8401, 0.6155]\", \"lift_point\": \"[0.8401, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0338, 0.4432]\", \"lift_point\": \"[0.0338, 0.4432]\", \"typed_text\": \"\" Goal: What's a good restaurant in Philadelphia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Philadelphia?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14844_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8401, 0.6155]\", \"lift_point\": \"[0.8401, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0338, 0.4432]\", \"lift_point\": \"[0.0338, 0.4432]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Philadelphia?\" Goal: What's a good restaurant in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.2706]\", \"lift_point\": \"[0.1147, 0.2706]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14844_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8401, 0.6155]\", \"lift_point\": \"[0.8401, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0338, 0.4432]\", \"lift_point\": \"[0.0338, 0.4432]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Philadelphia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.2706]\", \"lift_point\": \"[0.1147, 0.2706]\", \"typed_text\": \"\" Goal: What's a good restaurant in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.2453]\", \"lift_point\": \"[0.4472, 0.2453]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14844_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8401, 0.6155]\", \"lift_point\": \"[0.8401, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0338, 0.4432]\", \"lift_point\": \"[0.0338, 0.4432]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Philadelphia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.2706]\", \"lift_point\": \"[0.1147, 0.2706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.2453]\", \"lift_point\": \"[0.4472, 0.2453]\", \"typed_text\": \"\" Goal: What's a good restaurant in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.809, 0.4258]\", \"lift_point\": \"[0.809, 0.4258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14844_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8401, 0.6155]\", \"lift_point\": \"[0.8401, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0338, 0.4432]\", \"lift_point\": \"[0.0338, 0.4432]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Philadelphia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.2706]\", \"lift_point\": \"[0.1147, 0.2706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.2453]\", \"lift_point\": \"[0.4472, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.809, 0.4258]\", \"lift_point\": \"[0.809, 0.4258]\", \"typed_text\": \"\" Goal: What's a good restaurant in Philadelphia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/14844_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15042_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.665]\", \"lift_point\": \"[0.7845, 0.665]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15042_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.665]\", \"lift_point\": \"[0.7845, 0.665]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.3483]\", \"lift_point\": \"[0.0706, 0.3483]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15042_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.665]\", \"lift_point\": \"[0.7845, 0.665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.3483]\", \"lift_point\": \"[0.0706, 0.3483]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15042_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.665]\", \"lift_point\": \"[0.7845, 0.665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.3483]\", \"lift_point\": \"[0.0706, 0.3483]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1437, 0.6097]\", \"lift_point\": \"[0.1437, 0.6097]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15042_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.665]\", \"lift_point\": \"[0.7845, 0.665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.3483]\", \"lift_point\": \"[0.0706, 0.3483]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1437, 0.6097]\", \"lift_point\": \"[0.1437, 0.6097]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15042_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.7258]\", \"lift_point\": \"[0.7861, 0.7258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15070_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.7258]\", \"lift_point\": \"[0.7861, 0.7258]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.4528]\", \"lift_point\": \"[0.0557, 0.4528]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15070_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.7258]\", \"lift_point\": \"[0.7861, 0.7258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.4528]\", \"lift_point\": \"[0.0557, 0.4528]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15070_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.7258]\", \"lift_point\": \"[0.7861, 0.7258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.4528]\", \"lift_point\": \"[0.0557, 0.4528]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1416, 0.6371]\", \"lift_point\": \"[0.1416, 0.6371]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15070_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.7258]\", \"lift_point\": \"[0.7861, 0.7258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.4528]\", \"lift_point\": \"[0.0557, 0.4528]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1416, 0.6371]\", \"lift_point\": \"[0.1416, 0.6371]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15070_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7716, 0.67]\", \"lift_point\": \"[0.7716, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8312, 0.4462]\", \"lift_point\": \"[0.8312, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3808, 0.4894]\", \"lift_point\": \"[0.3808, 0.4894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4896]\", \"lift_point\": \"[0.3952, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3421, 0.5452]\", \"lift_point\": \"[0.3421, 0.5452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.497]\", \"lift_point\": \"[0.3782, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1705, 0.6327]\", \"lift_point\": \"[0.1705, 0.6327]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0636, 0.2848]\", \"lift_point\": \"[0.0636, 0.2848]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1705, 0.6327]\", \"lift_point\": \"[0.1705, 0.6327]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0415, 0.6002]\", \"lift_point\": \"[0.0415, 0.6002]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Los Angeles?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1705, 0.6327]\", \"lift_point\": \"[0.1705, 0.6327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0415, 0.6002]\", \"lift_point\": \"[0.0415, 0.6002]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1091, 0.4122]\", \"lift_point\": \"[0.1091, 0.4122]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1705, 0.6327]\", \"lift_point\": \"[0.1705, 0.6327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0415, 0.6002]\", \"lift_point\": \"[0.0415, 0.6002]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1091, 0.4122]\", \"lift_point\": \"[0.1091, 0.4122]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.6363]\", \"lift_point\": \"[0.2208, 0.6363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.411, 0.1532]\", \"lift_point\": \"[0.411, 0.1532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1705, 0.6327]\", \"lift_point\": \"[0.1705, 0.6327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0415, 0.6002]\", \"lift_point\": \"[0.0415, 0.6002]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1091, 0.4122]\", \"lift_point\": \"[0.1091, 0.4122]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.6363]\", \"lift_point\": \"[0.2208, 0.6363]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15182_19.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.7148]\", \"lift_point\": \"[0.7998, 0.7148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15254_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.7148]\", \"lift_point\": \"[0.7998, 0.7148]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.5005]\", \"lift_point\": \"[0.0618, 0.5005]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15254_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.7148]\", \"lift_point\": \"[0.7998, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.5005]\", \"lift_point\": \"[0.0618, 0.5005]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15254_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.7148]\", \"lift_point\": \"[0.7998, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.5005]\", \"lift_point\": \"[0.0618, 0.5005]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.7127]\", \"lift_point\": \"[0.0627, 0.7127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15254_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.7148]\", \"lift_point\": \"[0.7998, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.5005]\", \"lift_point\": \"[0.0618, 0.5005]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.7127]\", \"lift_point\": \"[0.0627, 0.7127]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3877]\", \"lift_point\": \"[0.1329, 0.3877]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15254_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.7148]\", \"lift_point\": \"[0.7998, 0.7148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.5005]\", \"lift_point\": \"[0.0618, 0.5005]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.7127]\", \"lift_point\": \"[0.0627, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3877]\", \"lift_point\": \"[0.1329, 0.3877]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15254_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15278_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6818]\", \"lift_point\": \"[0.7963, 0.6818]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15278_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6818]\", \"lift_point\": \"[0.7963, 0.6818]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.4833]\", \"lift_point\": \"[0.0617, 0.4833]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15278_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6818]\", \"lift_point\": \"[0.7963, 0.6818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.4833]\", \"lift_point\": \"[0.0617, 0.4833]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach tent?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15278_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6818]\", \"lift_point\": \"[0.7963, 0.6818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.4833]\", \"lift_point\": \"[0.0617, 0.4833]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach tent?\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.6532]\", \"lift_point\": \"[0.0524, 0.6532]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15278_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6818]\", \"lift_point\": \"[0.7963, 0.6818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.4833]\", \"lift_point\": \"[0.0617, 0.4833]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach tent?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.6532]\", \"lift_point\": \"[0.0524, 0.6532]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1299, 0.3755]\", \"lift_point\": \"[0.1299, 0.3755]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15278_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6818]\", \"lift_point\": \"[0.7963, 0.6818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.4833]\", \"lift_point\": \"[0.0617, 0.4833]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach tent?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.6532]\", \"lift_point\": \"[0.0524, 0.6532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1299, 0.3755]\", \"lift_point\": \"[0.1299, 0.3755]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15278_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1254]\", \"lift_point\": \"[0.793, 0.1254]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1254]\", \"lift_point\": \"[0.793, 0.1254]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.5311]\", \"lift_point\": \"[0.1898, 0.5311]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1254]\", \"lift_point\": \"[0.793, 0.1254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.5311]\", \"lift_point\": \"[0.1898, 0.5311]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4743]\", \"lift_point\": \"[0.061, 0.4743]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1254]\", \"lift_point\": \"[0.793, 0.1254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.5311]\", \"lift_point\": \"[0.1898, 0.5311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4743]\", \"lift_point\": \"[0.061, 0.4743]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1254]\", \"lift_point\": \"[0.793, 0.1254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.5311]\", \"lift_point\": \"[0.1898, 0.5311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4743]\", \"lift_point\": \"[0.061, 0.4743]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.706]\", \"lift_point\": \"[0.07, 0.706]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1254]\", \"lift_point\": \"[0.793, 0.1254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.5311]\", \"lift_point\": \"[0.1898, 0.5311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4743]\", \"lift_point\": \"[0.061, 0.4743]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.706]\", \"lift_point\": \"[0.07, 0.706]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1312, 0.4735]\", \"lift_point\": \"[0.1312, 0.4735]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1254]\", \"lift_point\": \"[0.793, 0.1254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.5311]\", \"lift_point\": \"[0.1898, 0.5311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4743]\", \"lift_point\": \"[0.061, 0.4743]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.706]\", \"lift_point\": \"[0.07, 0.706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1312, 0.4735]\", \"lift_point\": \"[0.1312, 0.4735]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15431_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2456, 0.4749]\", \"lift_point\": \"[0.2456, 0.4749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15438_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2456, 0.4749]\", \"lift_point\": \"[0.2456, 0.4749]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0333, 0.5603]\", \"lift_point\": \"[0.0333, 0.5603]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15438_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2456, 0.4749]\", \"lift_point\": \"[0.2456, 0.4749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0333, 0.5603]\", \"lift_point\": \"[0.0333, 0.5603]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.418, 0.719]\", \"lift_point\": \"[0.418, 0.719]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15438_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2456, 0.4749]\", \"lift_point\": \"[0.2456, 0.4749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0333, 0.5603]\", \"lift_point\": \"[0.0333, 0.5603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.418, 0.719]\", \"lift_point\": \"[0.418, 0.719]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.9279]\", \"lift_point\": \"[0.0567, 0.9279]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15438_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2456, 0.4749]\", \"lift_point\": \"[0.2456, 0.4749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0333, 0.5603]\", \"lift_point\": \"[0.0333, 0.5603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.418, 0.719]\", \"lift_point\": \"[0.418, 0.719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.9279]\", \"lift_point\": \"[0.0567, 0.9279]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.9291]\", \"lift_point\": \"[0.0597, 0.9291]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15438_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2456, 0.4749]\", \"lift_point\": \"[0.2456, 0.4749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0333, 0.5603]\", \"lift_point\": \"[0.0333, 0.5603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.418, 0.719]\", \"lift_point\": \"[0.418, 0.719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.9279]\", \"lift_point\": \"[0.0567, 0.9279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.9291]\", \"lift_point\": \"[0.0597, 0.9291]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1702, 0.5937]\", \"lift_point\": \"[0.1702, 0.5937]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15438_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2456, 0.4749]\", \"lift_point\": \"[0.2456, 0.4749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0333, 0.5603]\", \"lift_point\": \"[0.0333, 0.5603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.418, 0.719]\", \"lift_point\": \"[0.418, 0.719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.9279]\", \"lift_point\": \"[0.0567, 0.9279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.9291]\", \"lift_point\": \"[0.0597, 0.9291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1702, 0.5937]\", \"lift_point\": \"[0.1702, 0.5937]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15438_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's US dollar exchange rate against the South Korean Won?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.4314]\", \"lift_point\": \"[0.907, 0.4314]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15488_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.4314]\", \"lift_point\": \"[0.907, 0.4314]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the South Korean Won?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rate against the South Korean Won?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15488_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.4314]\", \"lift_point\": \"[0.907, 0.4314]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rate against the South Korean Won?\" Goal: What's US dollar exchange rate against the South Korean Won?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.132, 0.4854]\", \"lift_point\": \"[0.132, 0.4854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15488_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.4314]\", \"lift_point\": \"[0.907, 0.4314]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rate against the South Korean Won?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.132, 0.4854]\", \"lift_point\": \"[0.132, 0.4854]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the South Korean Won?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15488_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8858, 0.3312]\", \"lift_point\": \"[0.8858, 0.3312]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15573_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8858, 0.3312]\", \"lift_point\": \"[0.8858, 0.3312]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.3928]\", \"lift_point\": \"[0.0637, 0.3928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15573_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8858, 0.3312]\", \"lift_point\": \"[0.8858, 0.3312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.3928]\", \"lift_point\": \"[0.0637, 0.3928]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15573_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8858, 0.3312]\", \"lift_point\": \"[0.8858, 0.3312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.3928]\", \"lift_point\": \"[0.0637, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.5734]\", \"lift_point\": \"[0.1337, 0.5734]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15573_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8858, 0.3312]\", \"lift_point\": \"[0.8858, 0.3312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.3928]\", \"lift_point\": \"[0.0637, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.5734]\", \"lift_point\": \"[0.1337, 0.5734]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15573_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8106, 0.6268]\", \"lift_point\": \"[0.8106, 0.6268]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15577_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8106, 0.6268]\", \"lift_point\": \"[0.8106, 0.6268]\", \"typed_text\": \"\" Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0451, 0.9261]\", \"lift_point\": \"[0.0451, 0.9261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15577_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8106, 0.6268]\", \"lift_point\": \"[0.8106, 0.6268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0451, 0.9261]\", \"lift_point\": \"[0.0451, 0.9261]\", \"typed_text\": \"\" Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1713, 0.5737]\", \"lift_point\": \"[0.1713, 0.5737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15577_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8106, 0.6268]\", \"lift_point\": \"[0.8106, 0.6268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0451, 0.9261]\", \"lift_point\": \"[0.0451, 0.9261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1713, 0.5737]\", \"lift_point\": \"[0.1713, 0.5737]\", \"typed_text\": \"\" Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15577_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.4322]\", \"lift_point\": \"[0.882, 0.4322]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15726_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.4322]\", \"lift_point\": \"[0.882, 0.4322]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.4232]\", \"lift_point\": \"[0.0616, 0.4232]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15726_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.4322]\", \"lift_point\": \"[0.882, 0.4322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.4232]\", \"lift_point\": \"[0.0616, 0.4232]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new perfume\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15726_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.4322]\", \"lift_point\": \"[0.882, 0.4322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.4232]\", \"lift_point\": \"[0.0616, 0.4232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new perfume\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.3726]\", \"lift_point\": \"[0.777, 0.3726]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15726_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.4322]\", \"lift_point\": \"[0.882, 0.4322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.4232]\", \"lift_point\": \"[0.0616, 0.4232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new perfume\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.3726]\", \"lift_point\": \"[0.777, 0.3726]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1158, 0.3405]\", \"lift_point\": \"[0.1158, 0.3405]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15726_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.4322]\", \"lift_point\": \"[0.882, 0.4322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.4232]\", \"lift_point\": \"[0.0616, 0.4232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new perfume\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.3726]\", \"lift_point\": \"[0.777, 0.3726]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1158, 0.3405]\", \"lift_point\": \"[0.1158, 0.3405]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15726_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6348, 0.8499]\", \"lift_point\": \"[0.6348, 0.8499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6392, 0.8663]\", \"lift_point\": \"[0.6392, 0.8663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0508, 0.0719]\", \"lift_point\": \"[0.0508, 0.0719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0437, 0.8153]\", \"lift_point\": \"[0.0437, 0.8153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3804]\", \"lift_point\": \"[0.0634, 0.3804]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot Reviews?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8578, 0.9498]\", \"lift_point\": \"[0.8578, 0.9498]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2834, 0.3351]\", \"lift_point\": \"[0.2834, 0.3351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8578, 0.9498]\", \"lift_point\": \"[0.8578, 0.9498]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8578, 0.9498]\", \"lift_point\": \"[0.8578, 0.9498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6896, 0.3524]\", \"lift_point\": \"[0.6896, 0.3524]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8578, 0.9498]\", \"lift_point\": \"[0.8578, 0.9498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6896, 0.3524]\", \"lift_point\": \"[0.6896, 0.3524]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.2445]\", \"lift_point\": \"[0.3458, 0.2445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8578, 0.9498]\", \"lift_point\": \"[0.8578, 0.9498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6896, 0.3524]\", \"lift_point\": \"[0.6896, 0.3524]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.2445]\", \"lift_point\": \"[0.3458, 0.2445]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15764_19.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.8463]\", \"lift_point\": \"[0.0577, 0.8463]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.8463]\", \"lift_point\": \"[0.0577, 0.8463]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.0711]\", \"lift_point\": \"[0.0481, 0.0711]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.8463]\", \"lift_point\": \"[0.0577, 0.8463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.0711]\", \"lift_point\": \"[0.0481, 0.0711]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.8343]\", \"lift_point\": \"[0.0464, 0.8343]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.8463]\", \"lift_point\": \"[0.0577, 0.8463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.0711]\", \"lift_point\": \"[0.0481, 0.0711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.8343]\", \"lift_point\": \"[0.0464, 0.8343]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.5325]\", \"lift_point\": \"[0.0624, 0.5325]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.8463]\", \"lift_point\": \"[0.0577, 0.8463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.0711]\", \"lift_point\": \"[0.0481, 0.0711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.8343]\", \"lift_point\": \"[0.0464, 0.8343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.5325]\", \"lift_point\": \"[0.0624, 0.5325]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameTrailers\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.8463]\", \"lift_point\": \"[0.0577, 0.8463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.0711]\", \"lift_point\": \"[0.0481, 0.0711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.8343]\", \"lift_point\": \"[0.0464, 0.8343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.5325]\", \"lift_point\": \"[0.0624, 0.5325]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameTrailers\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.3623]\", \"lift_point\": \"[0.1229, 0.3623]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5986, 0.8539]\", \"lift_point\": \"[0.5986, 0.8539]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.0591]\", \"lift_point\": \"[0.0633, 0.0591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.8463]\", \"lift_point\": \"[0.0577, 0.8463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.0711]\", \"lift_point\": \"[0.0481, 0.0711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.8343]\", \"lift_point\": \"[0.0464, 0.8343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.5325]\", \"lift_point\": \"[0.0624, 0.5325]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameTrailers\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.3623]\", \"lift_point\": \"[0.1229, 0.3623]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15781_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in the Bahamas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8813, 0.3494]\", \"lift_point\": \"[0.8813, 0.3494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15922_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8813, 0.3494]\", \"lift_point\": \"[0.8813, 0.3494]\", \"typed_text\": \"\" Goal: What's the news in the Bahamas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0436, 0.3128]\", \"lift_point\": \"[0.0436, 0.3128]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15922_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8813, 0.3494]\", \"lift_point\": \"[0.8813, 0.3494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0436, 0.3128]\", \"lift_point\": \"[0.0436, 0.3128]\", \"typed_text\": \"\" Goal: What's the news in the Bahamas?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Bahamas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15922_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8813, 0.3494]\", \"lift_point\": \"[0.8813, 0.3494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0436, 0.3128]\", \"lift_point\": \"[0.0436, 0.3128]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Bahamas\" Goal: What's the news in the Bahamas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3599, 0.3964]\", \"lift_point\": \"[0.3599, 0.3964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15922_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8813, 0.3494]\", \"lift_point\": \"[0.8813, 0.3494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0436, 0.3128]\", \"lift_point\": \"[0.0436, 0.3128]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Bahamas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3599, 0.3964]\", \"lift_point\": \"[0.3599, 0.3964]\", \"typed_text\": \"\" Goal: What's the news in the Bahamas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15922_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15928_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8874, 0.4462]\", \"lift_point\": \"[0.8874, 0.4462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15928_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8874, 0.4462]\", \"lift_point\": \"[0.8874, 0.4462]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.4272]\", \"lift_point\": \"[0.0649, 0.4272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15928_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8874, 0.4462]\", \"lift_point\": \"[0.8874, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.4272]\", \"lift_point\": \"[0.0649, 0.4272]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15928_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8874, 0.4462]\", \"lift_point\": \"[0.8874, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.4272]\", \"lift_point\": \"[0.0649, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam?\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1177, 0.2747]\", \"lift_point\": \"[0.1177, 0.2747]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15928_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8874, 0.4462]\", \"lift_point\": \"[0.8874, 0.4462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.4272]\", \"lift_point\": \"[0.0649, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1177, 0.2747]\", \"lift_point\": \"[0.1177, 0.2747]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/15928_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16291_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9082, 0.3476]\", \"lift_point\": \"[0.9082, 0.3476]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16291_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9082, 0.3476]\", \"lift_point\": \"[0.9082, 0.3476]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8854, 0.439]\", \"lift_point\": \"[0.8854, 0.439]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16291_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9082, 0.3476]\", \"lift_point\": \"[0.9082, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8854, 0.439]\", \"lift_point\": \"[0.8854, 0.439]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.2781]\", \"lift_point\": \"[0.0666, 0.2781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16291_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9082, 0.3476]\", \"lift_point\": \"[0.9082, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8854, 0.439]\", \"lift_point\": \"[0.8854, 0.439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.2781]\", \"lift_point\": \"[0.0666, 0.2781]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16291_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9082, 0.3476]\", \"lift_point\": \"[0.9082, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8854, 0.439]\", \"lift_point\": \"[0.8854, 0.439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.2781]\", \"lift_point\": \"[0.0666, 0.2781]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Chicago?\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1803, 0.4498]\", \"lift_point\": \"[0.1803, 0.4498]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16291_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9082, 0.3476]\", \"lift_point\": \"[0.9082, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8854, 0.439]\", \"lift_point\": \"[0.8854, 0.439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.2781]\", \"lift_point\": \"[0.0666, 0.2781]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1803, 0.4498]\", \"lift_point\": \"[0.1803, 0.4498]\", \"typed_text\": \"\" Goal: What's the weather like in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16291_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16596_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16596_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3663, 0.1218]\", \"lift_point\": \"[0.3663, 0.1218]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16596_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3663, 0.1218]\", \"lift_point\": \"[0.3663, 0.1218]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16596_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5203]\", \"lift_point\": \"[0.876, 0.5203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5203]\", \"lift_point\": \"[0.876, 0.5203]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.603, 0.3919]\", \"lift_point\": \"[0.603, 0.3919]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5203]\", \"lift_point\": \"[0.876, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.603, 0.3919]\", \"lift_point\": \"[0.603, 0.3919]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4228, 0.5073]\", \"lift_point\": \"[0.4228, 0.5073]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5203]\", \"lift_point\": \"[0.876, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.603, 0.3919]\", \"lift_point\": \"[0.603, 0.3919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4228, 0.5073]\", \"lift_point\": \"[0.4228, 0.5073]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3074, 0.7011]\", \"lift_point\": \"[0.3074, 0.7011]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5203]\", \"lift_point\": \"[0.876, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.603, 0.3919]\", \"lift_point\": \"[0.603, 0.3919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4228, 0.5073]\", \"lift_point\": \"[0.4228, 0.5073]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3074, 0.7011]\", \"lift_point\": \"[0.3074, 0.7011]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6771, 0.7448]\", \"lift_point\": \"[0.6771, 0.7448]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5203]\", \"lift_point\": \"[0.876, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.603, 0.3919]\", \"lift_point\": \"[0.603, 0.3919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4228, 0.5073]\", \"lift_point\": \"[0.4228, 0.5073]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3074, 0.7011]\", \"lift_point\": \"[0.3074, 0.7011]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6771, 0.7448]\", \"lift_point\": \"[0.6771, 0.7448]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4335, 0.8971]\", \"lift_point\": \"[0.4335, 0.8971]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3585, 0.6635]\", \"lift_point\": \"[0.3585, 0.6635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.149]\", \"lift_point\": \"[0.0714, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5203]\", \"lift_point\": \"[0.876, 0.5203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.603, 0.3919]\", \"lift_point\": \"[0.603, 0.3919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4228, 0.5073]\", \"lift_point\": \"[0.4228, 0.5073]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3074, 0.7011]\", \"lift_point\": \"[0.3074, 0.7011]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6771, 0.7448]\", \"lift_point\": \"[0.6771, 0.7448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4335, 0.8971]\", \"lift_point\": \"[0.4335, 0.8971]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16673_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.5237]\", \"lift_point\": \"[0.8827, 0.5237]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16839_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.5237]\", \"lift_point\": \"[0.8827, 0.5237]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16839_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.5237]\", \"lift_point\": \"[0.8827, 0.5237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" at home depot\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16839_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.5237]\", \"lift_point\": \"[0.8827, 0.5237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" at home depot\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2118, 0.495]\", \"lift_point\": \"[0.2118, 0.495]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16839_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.5237]\", \"lift_point\": \"[0.8827, 0.5237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" at home depot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2118, 0.495]\", \"lift_point\": \"[0.2118, 0.495]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16839_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.5142]\", \"lift_point\": \"[0.3531, 0.5142]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.5142]\", \"lift_point\": \"[0.3531, 0.5142]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.5142]\", \"lift_point\": \"[0.3531, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3485, 0.2759]\", \"lift_point\": \"[0.3485, 0.2759]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.5142]\", \"lift_point\": \"[0.3531, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3485, 0.2759]\", \"lift_point\": \"[0.3485, 0.2759]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.48]\", \"lift_point\": \"[0.0475, 0.48]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.5142]\", \"lift_point\": \"[0.3531, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3485, 0.2759]\", \"lift_point\": \"[0.3485, 0.2759]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.48]\", \"lift_point\": \"[0.0475, 0.48]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.5142]\", \"lift_point\": \"[0.3531, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3485, 0.2759]\", \"lift_point\": \"[0.3485, 0.2759]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.48]\", \"lift_point\": \"[0.0475, 0.48]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.5142]\", \"lift_point\": \"[0.3531, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3485, 0.2759]\", \"lift_point\": \"[0.3485, 0.2759]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.48]\", \"lift_point\": \"[0.0475, 0.48]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16863_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.159, 0.4774]\", \"lift_point\": \"[0.159, 0.4774]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.159, 0.4774]\", \"lift_point\": \"[0.159, 0.4774]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5075, 0.042]\", \"lift_point\": \"[0.5075, 0.042]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.4784]\", \"lift_point\": \"[0.7715, 0.4784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.159, 0.4774]\", \"lift_point\": \"[0.159, 0.4774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5075, 0.042]\", \"lift_point\": \"[0.5075, 0.042]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.3077]\", \"lift_point\": \"[0.1134, 0.3077]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.8065]\", \"lift_point\": \"[0.0461, 0.8065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.159, 0.4774]\", \"lift_point\": \"[0.159, 0.4774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5075, 0.042]\", \"lift_point\": \"[0.5075, 0.042]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.3077]\", \"lift_point\": \"[0.1134, 0.3077]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ariana Grande \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.159, 0.4774]\", \"lift_point\": \"[0.159, 0.4774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5075, 0.042]\", \"lift_point\": \"[0.5075, 0.042]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.3077]\", \"lift_point\": \"[0.1134, 0.3077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.159, 0.4774]\", \"lift_point\": \"[0.159, 0.4774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5075, 0.042]\", \"lift_point\": \"[0.5075, 0.042]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.3077]\", \"lift_point\": \"[0.1134, 0.3077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2203]\", \"lift_point\": \"[0.173, 0.2203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7711, 0.2952]\", \"lift_point\": \"[0.7711, 0.2952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.159, 0.4774]\", \"lift_point\": \"[0.159, 0.4774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5075, 0.042]\", \"lift_point\": \"[0.5075, 0.042]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.3077]\", \"lift_point\": \"[0.1134, 0.3077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1774, 0.2732]\", \"lift_point\": \"[0.1774, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2203]\", \"lift_point\": \"[0.173, 0.2203]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16897_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" rent for in Portland?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" rent for in Portland?\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" rent for in Portland?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" rent for in Portland?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4963, 0.3306]\", \"lift_point\": \"[0.4963, 0.3306]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3458, 0.485]\", \"lift_point\": \"[0.3458, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4301]\", \"lift_point\": \"[0.0518, 0.4301]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" rent for in Portland?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4963, 0.3306]\", \"lift_point\": \"[0.4963, 0.3306]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16907_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3506, 0.4922]\", \"lift_point\": \"[0.3506, 0.4922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3506, 0.4922]\", \"lift_point\": \"[0.3506, 0.4922]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5814]\", \"lift_point\": \"[0.0503, 0.5814]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3506, 0.4922]\", \"lift_point\": \"[0.3506, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5814]\", \"lift_point\": \"[0.0503, 0.5814]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3506, 0.4922]\", \"lift_point\": \"[0.3506, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5814]\", \"lift_point\": \"[0.0503, 0.5814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3506, 0.4922]\", \"lift_point\": \"[0.3506, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5814]\", \"lift_point\": \"[0.0503, 0.5814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3506, 0.4922]\", \"lift_point\": \"[0.3506, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5814]\", \"lift_point\": \"[0.0503, 0.5814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.4504]\", \"lift_point\": \"[0.3179, 0.4504]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3506, 0.4922]\", \"lift_point\": \"[0.3506, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5814]\", \"lift_point\": \"[0.0503, 0.5814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3179, 0.4504]\", \"lift_point\": \"[0.3179, 0.4504]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/16946_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17031_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3143, 0.5054]\", \"lift_point\": \"[0.3143, 0.5054]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17031_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3143, 0.5054]\", \"lift_point\": \"[0.3143, 0.5054]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0659, 0.5205]\", \"lift_point\": \"[0.0659, 0.5205]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17031_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3143, 0.5054]\", \"lift_point\": \"[0.3143, 0.5054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0659, 0.5205]\", \"lift_point\": \"[0.0659, 0.5205]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17031_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3143, 0.5054]\", \"lift_point\": \"[0.3143, 0.5054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0659, 0.5205]\", \"lift_point\": \"[0.0659, 0.5205]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17031_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3143, 0.5054]\", \"lift_point\": \"[0.3143, 0.5054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0659, 0.5205]\", \"lift_point\": \"[0.0659, 0.5205]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17031_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of France?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17114_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17114_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.5034]\", \"lift_point\": \"[0.3288, 0.5034]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17114_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.5034]\", \"lift_point\": \"[0.3288, 0.5034]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2554, 0.5853]\", \"lift_point\": \"[0.2554, 0.5853]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17114_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.5034]\", \"lift_point\": \"[0.3288, 0.5034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2554, 0.5853]\", \"lift_point\": \"[0.2554, 0.5853]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17114_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.5034]\", \"lift_point\": \"[0.3288, 0.5034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2554, 0.5853]\", \"lift_point\": \"[0.2554, 0.5853]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17114_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.5034]\", \"lift_point\": \"[0.3288, 0.5034]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2554, 0.5853]\", \"lift_point\": \"[0.2554, 0.5853]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17114_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7828, 0.6952]\", \"lift_point\": \"[0.7828, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8128, 0.4628]\", \"lift_point\": \"[0.8128, 0.4628]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8954, 0.5411]\", \"lift_point\": \"[0.8954, 0.5411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.099]\", \"lift_point\": \"[0.8974, 0.099]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.305, 0.6055]\", \"lift_point\": \"[0.305, 0.6055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8128, 0.4628]\", \"lift_point\": \"[0.8128, 0.4628]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17218_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7197, 0.5707]\", \"lift_point\": \"[0.7197, 0.5707]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7197, 0.5707]\", \"lift_point\": \"[0.7197, 0.5707]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.5328]\", \"lift_point\": \"[0.9042, 0.5328]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.699]\", \"lift_point\": \"[0.7737, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7197, 0.5707]\", \"lift_point\": \"[0.7197, 0.5707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.5328]\", \"lift_point\": \"[0.9042, 0.5328]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0621, 0.536]\", \"lift_point\": \"[0.0621, 0.536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7197, 0.5707]\", \"lift_point\": \"[0.7197, 0.5707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.5328]\", \"lift_point\": \"[0.9042, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5991, 0.2031]\", \"lift_point\": \"[0.5991, 0.2031]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.7654]\", \"lift_point\": \"[0.1411, 0.7654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7197, 0.5707]\", \"lift_point\": \"[0.7197, 0.5707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.5328]\", \"lift_point\": \"[0.9042, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5991, 0.2031]\", \"lift_point\": \"[0.5991, 0.2031]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17269_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.3694]\", \"lift_point\": \"[0.1317, 0.3694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.3694]\", \"lift_point\": \"[0.1317, 0.3694]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.3694]\", \"lift_point\": \"[0.1317, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7844, 0.4128]\", \"lift_point\": \"[0.7844, 0.4128]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.3694]\", \"lift_point\": \"[0.1317, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7844, 0.4128]\", \"lift_point\": \"[0.7844, 0.4128]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2666, 0.9496]\", \"lift_point\": \"[0.2666, 0.9496]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.3694]\", \"lift_point\": \"[0.1317, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7844, 0.4128]\", \"lift_point\": \"[0.7844, 0.4128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2666, 0.9496]\", \"lift_point\": \"[0.2666, 0.9496]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.5883]\", \"lift_point\": \"[0.0667, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.3694]\", \"lift_point\": \"[0.1317, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7844, 0.4128]\", \"lift_point\": \"[0.7844, 0.4128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2666, 0.9496]\", \"lift_point\": \"[0.2666, 0.9496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7461, 0.9405]\", \"lift_point\": \"[0.7461, 0.9405]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the LG TV\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.3694]\", \"lift_point\": \"[0.1317, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7844, 0.4128]\", \"lift_point\": \"[0.7844, 0.4128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2666, 0.9496]\", \"lift_point\": \"[0.2666, 0.9496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7461, 0.9405]\", \"lift_point\": \"[0.7461, 0.9405]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17509_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1467, 0.5544]\", \"lift_point\": \"[0.1467, 0.5544]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1467, 0.5544]\", \"lift_point\": \"[0.1467, 0.5544]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1467, 0.5544]\", \"lift_point\": \"[0.1467, 0.5544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5362, 0.4795]\", \"lift_point\": \"[0.5362, 0.4795]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1467, 0.5544]\", \"lift_point\": \"[0.1467, 0.5544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5362, 0.4795]\", \"lift_point\": \"[0.5362, 0.4795]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6619]\", \"lift_point\": \"[0.7894, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1467, 0.5544]\", \"lift_point\": \"[0.1467, 0.5544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5362, 0.4795]\", \"lift_point\": \"[0.5362, 0.4795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6009, 0.1258]\", \"lift_point\": \"[0.6009, 0.1258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.3985]\", \"lift_point\": \"[0.1974, 0.3985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.5751]\", \"lift_point\": \"[0.064, 0.5751]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the new iPhone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1467, 0.5544]\", \"lift_point\": \"[0.1467, 0.5544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5362, 0.4795]\", \"lift_point\": \"[0.5362, 0.4795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6009, 0.1258]\", \"lift_point\": \"[0.6009, 0.1258]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17552_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.546]\", \"lift_point\": \"[0.0619, 0.546]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.546]\", \"lift_point\": \"[0.0619, 0.546]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.546]\", \"lift_point\": \"[0.0619, 0.546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.3932]\", \"lift_point\": \"[0.1341, 0.3932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.546]\", \"lift_point\": \"[0.0619, 0.546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.3932]\", \"lift_point\": \"[0.1341, 0.3932]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3713, 0.3599]\", \"lift_point\": \"[0.3713, 0.3599]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.546]\", \"lift_point\": \"[0.0619, 0.546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.3932]\", \"lift_point\": \"[0.1341, 0.3932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3713, 0.3599]\", \"lift_point\": \"[0.3713, 0.3599]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1276, 0.9579]\", \"lift_point\": \"[0.1276, 0.9579]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6931]\", \"lift_point\": \"[0.7808, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6936, 0.8393]\", \"lift_point\": \"[0.6936, 0.8393]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7686, 0.7016]\", \"lift_point\": \"[0.7686, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.546]\", \"lift_point\": \"[0.0619, 0.546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.3932]\", \"lift_point\": \"[0.1341, 0.3932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3713, 0.3599]\", \"lift_point\": \"[0.3713, 0.3599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1276, 0.9579]\", \"lift_point\": \"[0.1276, 0.9579]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17652_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.4752]\", \"lift_point\": \"[0.1236, 0.4752]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.4752]\", \"lift_point\": \"[0.1236, 0.4752]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.4752]\", \"lift_point\": \"[0.1236, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.4752]\", \"lift_point\": \"[0.1236, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5154, 0.5224]\", \"lift_point\": \"[0.5154, 0.5224]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.7048]\", \"lift_point\": \"[0.768, 0.7048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5235]\", \"lift_point\": \"[0.052, 0.5235]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.4752]\", \"lift_point\": \"[0.1236, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5154, 0.5224]\", \"lift_point\": \"[0.5154, 0.5224]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17669_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17747_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7754, 0.6913]\", \"lift_point\": \"[0.7754, 0.6913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17747_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7754, 0.6913]\", \"lift_point\": \"[0.7754, 0.6913]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4674]\", \"lift_point\": \"[0.0722, 0.4674]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17747_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7754, 0.6913]\", \"lift_point\": \"[0.7754, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4674]\", \"lift_point\": \"[0.0722, 0.4674]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Venezuela\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17747_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7754, 0.6913]\", \"lift_point\": \"[0.7754, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4674]\", \"lift_point\": \"[0.0722, 0.4674]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Venezuela\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.44]\", \"lift_point\": \"[0.1258, 0.44]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17747_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7754, 0.6913]\", \"lift_point\": \"[0.7754, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.4674]\", \"lift_point\": \"[0.0722, 0.4674]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Venezuela\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.44]\", \"lift_point\": \"[0.1258, 0.44]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17747_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.7931]\", \"lift_point\": \"[0.063, 0.7931]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.7931]\", \"lift_point\": \"[0.063, 0.7931]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.9342]\", \"lift_point\": \"[0.0644, 0.9342]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.7931]\", \"lift_point\": \"[0.063, 0.7931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.9342]\", \"lift_point\": \"[0.0644, 0.9342]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.5633]\", \"lift_point\": \"[0.0643, 0.5633]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.6747]\", \"lift_point\": \"[0.7785, 0.6747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.7931]\", \"lift_point\": \"[0.063, 0.7931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.9342]\", \"lift_point\": \"[0.0644, 0.9342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.5633]\", \"lift_point\": \"[0.0643, 0.5633]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9707, 0.5144]\", \"lift_point\": \"[0.9707, 0.5144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.7931]\", \"lift_point\": \"[0.063, 0.7931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.9342]\", \"lift_point\": \"[0.0644, 0.9342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.5633]\", \"lift_point\": \"[0.0643, 0.5633]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.2586]\", \"lift_point\": \"[0.1261, 0.2586]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.4917]\", \"lift_point\": \"[0.7778, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.7931]\", \"lift_point\": \"[0.063, 0.7931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.9342]\", \"lift_point\": \"[0.0644, 0.9342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.5633]\", \"lift_point\": \"[0.0643, 0.5633]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.2586]\", \"lift_point\": \"[0.1261, 0.2586]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1452, 0.8704]\", \"lift_point\": \"[0.1452, 0.8704]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5702, 0.2163]\", \"lift_point\": \"[0.5702, 0.2163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.5074]\", \"lift_point\": \"[0.7842, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.7931]\", \"lift_point\": \"[0.063, 0.7931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.9342]\", \"lift_point\": \"[0.0644, 0.9342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.5633]\", \"lift_point\": \"[0.0643, 0.5633]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.2586]\", \"lift_point\": \"[0.1261, 0.2586]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1452, 0.8704]\", \"lift_point\": \"[0.1452, 0.8704]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17765_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.4881]\", \"lift_point\": \"[0.0654, 0.4881]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.4881]\", \"lift_point\": \"[0.0654, 0.4881]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Seattle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.4881]\", \"lift_point\": \"[0.0654, 0.4881]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Seattle\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.3413]\", \"lift_point\": \"[0.1388, 0.3413]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.4881]\", \"lift_point\": \"[0.0654, 0.4881]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.3413]\", \"lift_point\": \"[0.1388, 0.3413]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.4881]\", \"lift_point\": \"[0.0654, 0.4881]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.3413]\", \"lift_point\": \"[0.1388, 0.3413]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.4881]\", \"lift_point\": \"[0.0654, 0.4881]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.3413]\", \"lift_point\": \"[0.1388, 0.3413]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8444, 0.5089]\", \"lift_point\": \"[0.8444, 0.5089]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7725, 0.6671]\", \"lift_point\": \"[0.7725, 0.6671]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.4881]\", \"lift_point\": \"[0.0654, 0.4881]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1388, 0.3413]\", \"lift_point\": \"[0.1388, 0.3413]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8444, 0.5089]\", \"lift_point\": \"[0.8444, 0.5089]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17832_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.863]\", \"lift_point\": \"[0.7915, 0.863]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.863]\", \"lift_point\": \"[0.7915, 0.863]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7662, 0.6751]\", \"lift_point\": \"[0.7662, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.863]\", \"lift_point\": \"[0.7915, 0.863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.674, 0.4341]\", \"lift_point\": \"[0.674, 0.4341]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0501, 0.3252]\", \"lift_point\": \"[0.0501, 0.3252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.863]\", \"lift_point\": \"[0.7915, 0.863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.674, 0.4341]\", \"lift_point\": \"[0.674, 0.4341]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6739, 0.4207]\", \"lift_point\": \"[0.6739, 0.4207]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.863]\", \"lift_point\": \"[0.7915, 0.863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.674, 0.4341]\", \"lift_point\": \"[0.674, 0.4341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6739, 0.4207]\", \"lift_point\": \"[0.6739, 0.4207]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4539, 0.447]\", \"lift_point\": \"[0.4539, 0.447]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1322, 0.2367]\", \"lift_point\": \"[0.1322, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7915, 0.863]\", \"lift_point\": \"[0.7915, 0.863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.674, 0.4341]\", \"lift_point\": \"[0.674, 0.4341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6739, 0.4207]\", \"lift_point\": \"[0.6739, 0.4207]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4539, 0.447]\", \"lift_point\": \"[0.4539, 0.447]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17916_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17926_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3937, 0.6838]\", \"lift_point\": \"[0.3937, 0.6838]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17926_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3937, 0.6838]\", \"lift_point\": \"[0.3937, 0.6838]\", \"typed_text\": \"\" Goal: What time is it?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17926_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6949]\", \"lift_point\": \"[0.7774, 0.6949]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17932_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6949]\", \"lift_point\": \"[0.7774, 0.6949]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.4845]\", \"lift_point\": \"[0.0724, 0.4845]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17932_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6949]\", \"lift_point\": \"[0.7774, 0.6949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.4845]\", \"lift_point\": \"[0.0724, 0.4845]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barbados\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17932_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6949]\", \"lift_point\": \"[0.7774, 0.6949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.4845]\", \"lift_point\": \"[0.0724, 0.4845]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barbados\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1338, 0.2837]\", \"lift_point\": \"[0.1338, 0.2837]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17932_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6949]\", \"lift_point\": \"[0.7774, 0.6949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.4845]\", \"lift_point\": \"[0.0724, 0.4845]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barbados\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1338, 0.2837]\", \"lift_point\": \"[0.1338, 0.2837]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4994, 0.7512]\", \"lift_point\": \"[0.4994, 0.7512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17932_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6949]\", \"lift_point\": \"[0.7774, 0.6949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.4845]\", \"lift_point\": \"[0.0724, 0.4845]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Barbados\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1338, 0.2837]\", \"lift_point\": \"[0.1338, 0.2837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4994, 0.7512]\", \"lift_point\": \"[0.4994, 0.7512]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/17932_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18263_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.1802]\", \"lift_point\": \"[0.5958, 0.1802]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18263_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.1802]\", \"lift_point\": \"[0.5958, 0.1802]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3673]\", \"lift_point\": \"[0.1279, 0.3673]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18263_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.1802]\", \"lift_point\": \"[0.5958, 0.1802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3673]\", \"lift_point\": \"[0.1279, 0.3673]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.8962]\", \"lift_point\": \"[0.0655, 0.8962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18263_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.1802]\", \"lift_point\": \"[0.5958, 0.1802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3673]\", \"lift_point\": \"[0.1279, 0.3673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.8962]\", \"lift_point\": \"[0.0655, 0.8962]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Greek restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18263_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.1802]\", \"lift_point\": \"[0.5958, 0.1802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3673]\", \"lift_point\": \"[0.1279, 0.3673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.8962]\", \"lift_point\": \"[0.0655, 0.8962]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Greek restaurants\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.6422]\", \"lift_point\": \"[0.1317, 0.6422]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18263_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5958, 0.1802]\", \"lift_point\": \"[0.5958, 0.1802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3673]\", \"lift_point\": \"[0.1279, 0.3673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.8962]\", \"lift_point\": \"[0.0655, 0.8962]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Greek restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1317, 0.6422]\", \"lift_point\": \"[0.1317, 0.6422]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18263_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1045, 0.5368]\", \"lift_point\": \"[0.1045, 0.5368]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1045, 0.5368]\", \"lift_point\": \"[0.1045, 0.5368]\", \"typed_text\": \"\" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0462, 0.8993]\", \"lift_point\": \"[0.0462, 0.8993]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1045, 0.5368]\", \"lift_point\": \"[0.1045, 0.5368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0462, 0.8993]\", \"lift_point\": \"[0.0462, 0.8993]\", \"typed_text\": \"\" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.4284]\", \"lift_point\": \"[0.0491, 0.4284]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1045, 0.5368]\", \"lift_point\": \"[0.1045, 0.5368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0462, 0.8993]\", \"lift_point\": \"[0.0462, 0.8993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.4284]\", \"lift_point\": \"[0.0491, 0.4284]\", \"typed_text\": \"\" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Beijing? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1045, 0.5368]\", \"lift_point\": \"[0.1045, 0.5368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0462, 0.8993]\", \"lift_point\": \"[0.0462, 0.8993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.4284]\", \"lift_point\": \"[0.0491, 0.4284]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Beijing? \" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1116, 0.4166]\", \"lift_point\": \"[0.1116, 0.4166]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5651, 0.1852]\", \"lift_point\": \"[0.5651, 0.1852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1045, 0.5368]\", \"lift_point\": \"[0.1045, 0.5368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0462, 0.8993]\", \"lift_point\": \"[0.0462, 0.8993]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.4284]\", \"lift_point\": \"[0.0491, 0.4284]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Beijing? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1116, 0.4166]\", \"lift_point\": \"[0.1116, 0.4166]\", \"typed_text\": \"\" Goal: What time is it in Beijing?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18294_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5386, 0.1224]\", \"lift_point\": \"[0.5386, 0.1224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3504, 0.3787]\", \"lift_point\": \"[0.3504, 0.3787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.5713]\", \"lift_point\": \"[0.1063, 0.5713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3504, 0.3787]\", \"lift_point\": \"[0.3504, 0.3787]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.9258]\", \"lift_point\": \"[0.0474, 0.9258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3504, 0.3787]\", \"lift_point\": \"[0.3504, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.444]\", \"lift_point\": \"[0.0606, 0.444]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3504, 0.3787]\", \"lift_point\": \"[0.3504, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the TCL TV cost? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3504, 0.3787]\", \"lift_point\": \"[0.3504, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1079, 0.3517]\", \"lift_point\": \"[0.1079, 0.3517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3504, 0.3787]\", \"lift_point\": \"[0.3504, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5519, 0.3016]\", \"lift_point\": \"[0.5519, 0.3016]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3504, 0.3787]\", \"lift_point\": \"[0.3504, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5519, 0.3016]\", \"lift_point\": \"[0.5519, 0.3016]\", \"typed_text\": \"\" Goal: How much does the TCL TV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18504_17.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1552]\", \"lift_point\": \"[0.5469, 0.1552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.5818]\", \"lift_point\": \"[0.0428, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.9008]\", \"lift_point\": \"[0.0428, 0.9008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0586, 0.3854]\", \"lift_point\": \"[0.0586, 0.3854]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.8195]\", \"lift_point\": \"[0.0533, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.8541]\", \"lift_point\": \"[0.0479, 0.8541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.8434]\", \"lift_point\": \"[0.0527, 0.8434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.8162]\", \"lift_point\": \"[0.0552, 0.8162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0326, 0.9083]\", \"lift_point\": \"[0.0326, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0512, 0.3511]\", \"lift_point\": \"[0.0512, 0.3511]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.9083]\", \"lift_point\": \"[0.0454, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0512, 0.3511]\", \"lift_point\": \"[0.0512, 0.3511]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Australian Dollar \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.3553]\", \"lift_point\": \"[0.0524, 0.3553]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0512, 0.3511]\", \"lift_point\": \"[0.0512, 0.3511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Australian Dollar \" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3218]\", \"lift_point\": \"[0.1125, 0.3218]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0512, 0.3511]\", \"lift_point\": \"[0.0512, 0.3511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3218]\", \"lift_point\": \"[0.1125, 0.3218]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.2897]\", \"lift_point\": \"[0.0664, 0.2897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0512, 0.3511]\", \"lift_point\": \"[0.0512, 0.3511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3218]\", \"lift_point\": \"[0.1125, 0.3218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7876, 0.173]\", \"lift_point\": \"[0.7876, 0.173]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0512, 0.3511]\", \"lift_point\": \"[0.0512, 0.3511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3218]\", \"lift_point\": \"[0.1125, 0.3218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7876, 0.173]\", \"lift_point\": \"[0.7876, 0.173]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3592, 0.8903]\", \"lift_point\": \"[0.3592, 0.8903]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0612, 0.5317]\", \"lift_point\": \"[0.0612, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.8925]\", \"lift_point\": \"[0.048, 0.8925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0512, 0.3511]\", \"lift_point\": \"[0.0512, 0.3511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Australian Dollar \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3218]\", \"lift_point\": \"[0.1125, 0.3218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7876, 0.173]\", \"lift_point\": \"[0.7876, 0.173]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3592, 0.8903]\", \"lift_point\": \"[0.3592, 0.8903]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Australian Dollar?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18530_26.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1066, 0.6427]\", \"lift_point\": \"[0.1066, 0.6427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1066, 0.6427]\", \"lift_point\": \"[0.1066, 0.6427]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.9284]\", \"lift_point\": \"[0.0473, 0.9284]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1066, 0.6427]\", \"lift_point\": \"[0.1066, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.9284]\", \"lift_point\": \"[0.0473, 0.9284]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4209]\", \"lift_point\": \"[0.0574, 0.4209]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1066, 0.6427]\", \"lift_point\": \"[0.1066, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.9284]\", \"lift_point\": \"[0.0473, 0.9284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4209]\", \"lift_point\": \"[0.0574, 0.4209]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1066, 0.6427]\", \"lift_point\": \"[0.1066, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.9284]\", \"lift_point\": \"[0.0473, 0.9284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4209]\", \"lift_point\": \"[0.0574, 0.4209]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires \" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1073, 0.2739]\", \"lift_point\": \"[0.1073, 0.2739]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1066, 0.6427]\", \"lift_point\": \"[0.1066, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.9284]\", \"lift_point\": \"[0.0473, 0.9284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4209]\", \"lift_point\": \"[0.0574, 0.4209]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1073, 0.2739]\", \"lift_point\": \"[0.1073, 0.2739]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2568, 0.2873]\", \"lift_point\": \"[0.2568, 0.2873]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5418, 0.1546]\", \"lift_point\": \"[0.5418, 0.1546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1066, 0.6427]\", \"lift_point\": \"[0.1066, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.9284]\", \"lift_point\": \"[0.0473, 0.9284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4209]\", \"lift_point\": \"[0.0574, 0.4209]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1073, 0.2739]\", \"lift_point\": \"[0.1073, 0.2739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2568, 0.2873]\", \"lift_point\": \"[0.2568, 0.2873]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18677_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1602]\", \"lift_point\": \"[0.5469, 0.1602]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1602]\", \"lift_point\": \"[0.5469, 0.1602]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.412]\", \"lift_point\": \"[0.1094, 0.412]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1602]\", \"lift_point\": \"[0.5469, 0.1602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.412]\", \"lift_point\": \"[0.1094, 0.412]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.92]\", \"lift_point\": \"[0.0442, 0.92]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1602]\", \"lift_point\": \"[0.5469, 0.1602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.412]\", \"lift_point\": \"[0.1094, 0.412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.92]\", \"lift_point\": \"[0.0442, 0.92]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0447, 0.3167]\", \"lift_point\": \"[0.0447, 0.3167]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1602]\", \"lift_point\": \"[0.5469, 0.1602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.412]\", \"lift_point\": \"[0.1094, 0.412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.92]\", \"lift_point\": \"[0.0442, 0.92]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0447, 0.3167]\", \"lift_point\": \"[0.0447, 0.3167]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Bolivia \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1602]\", \"lift_point\": \"[0.5469, 0.1602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.412]\", \"lift_point\": \"[0.1094, 0.412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.92]\", \"lift_point\": \"[0.0442, 0.92]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0447, 0.3167]\", \"lift_point\": \"[0.0447, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Bolivia \" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0988, 0.3816]\", \"lift_point\": \"[0.0988, 0.3816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.1602]\", \"lift_point\": \"[0.5469, 0.1602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.412]\", \"lift_point\": \"[0.1094, 0.412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0442, 0.92]\", \"lift_point\": \"[0.0442, 0.92]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0447, 0.3167]\", \"lift_point\": \"[0.0447, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Bolivia \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0988, 0.3816]\", \"lift_point\": \"[0.0988, 0.3816]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/18998_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5378, 0.1475]\", \"lift_point\": \"[0.5378, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5619, 0.1167]\", \"lift_point\": \"[0.5619, 0.1167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.9745]\", \"lift_point\": \"[0.1021, 0.9745]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.487]\", \"lift_point\": \"[0.0562, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.9745]\", \"lift_point\": \"[0.1021, 0.9745]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5861, 0.5358]\", \"lift_point\": \"[0.5861, 0.5358]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9112]\", \"lift_point\": \"[0.0534, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.9745]\", \"lift_point\": \"[0.1021, 0.9745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5861, 0.5358]\", \"lift_point\": \"[0.5861, 0.5358]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4939, 0.3748]\", \"lift_point\": \"[0.4939, 0.3748]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4799]\", \"lift_point\": \"[0.0619, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.9745]\", \"lift_point\": \"[0.1021, 0.9745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5861, 0.5358]\", \"lift_point\": \"[0.5861, 0.5358]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4939, 0.3748]\", \"lift_point\": \"[0.4939, 0.3748]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4843, 0.3748]\", \"lift_point\": \"[0.4843, 0.3748]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.9745]\", \"lift_point\": \"[0.1021, 0.9745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5861, 0.5358]\", \"lift_point\": \"[0.5861, 0.5358]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4939, 0.3748]\", \"lift_point\": \"[0.4939, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4843, 0.3748]\", \"lift_point\": \"[0.4843, 0.3748]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5964, 0.4444]\", \"lift_point\": \"[0.5964, 0.4444]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.4101]\", \"lift_point\": \"[0.1134, 0.4101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.9745]\", \"lift_point\": \"[0.1021, 0.9745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5861, 0.5358]\", \"lift_point\": \"[0.5861, 0.5358]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4939, 0.3748]\", \"lift_point\": \"[0.4939, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4843, 0.3748]\", \"lift_point\": \"[0.4843, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5964, 0.4444]\", \"lift_point\": \"[0.5964, 0.4444]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5567, 0.4671]\", \"lift_point\": \"[0.5567, 0.4671]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5912, 0.3496]\", \"lift_point\": \"[0.5912, 0.3496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9431, 0.52]\", \"lift_point\": \"[0.9431, 0.52]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.9745]\", \"lift_point\": \"[0.1021, 0.9745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5861, 0.5358]\", \"lift_point\": \"[0.5861, 0.5358]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4939, 0.3748]\", \"lift_point\": \"[0.4939, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4843, 0.3748]\", \"lift_point\": \"[0.4843, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5964, 0.4444]\", \"lift_point\": \"[0.5964, 0.4444]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5567, 0.4671]\", \"lift_point\": \"[0.5567, 0.4671]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19017_19.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0539, 0.3661]\", \"lift_point\": \"[0.0539, 0.3661]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0539, 0.3661]\", \"lift_point\": \"[0.0539, 0.3661]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.9027]\", \"lift_point\": \"[0.055, 0.9027]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0539, 0.3661]\", \"lift_point\": \"[0.0539, 0.3661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.9027]\", \"lift_point\": \"[0.055, 0.9027]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0275, 0.3803]\", \"lift_point\": \"[0.0275, 0.3803]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0539, 0.3661]\", \"lift_point\": \"[0.0539, 0.3661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.9027]\", \"lift_point\": \"[0.055, 0.9027]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0275, 0.3803]\", \"lift_point\": \"[0.0275, 0.3803]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Reviews? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0539, 0.3661]\", \"lift_point\": \"[0.0539, 0.3661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.9027]\", \"lift_point\": \"[0.055, 0.9027]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0275, 0.3803]\", \"lift_point\": \"[0.0275, 0.3803]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Reviews? \" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1097, 0.3554]\", \"lift_point\": \"[0.1097, 0.3554]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5412, 0.1542]\", \"lift_point\": \"[0.5412, 0.1542]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0539, 0.3661]\", \"lift_point\": \"[0.0539, 0.3661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.9027]\", \"lift_point\": \"[0.055, 0.9027]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0275, 0.3803]\", \"lift_point\": \"[0.0275, 0.3803]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Reviews? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1097, 0.3554]\", \"lift_point\": \"[0.1097, 0.3554]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Reviews?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19161_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Chile?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.3487]\", \"lift_point\": \"[0.1075, 0.3487]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.3487]\", \"lift_point\": \"[0.1075, 0.3487]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.3923]\", \"lift_point\": \"[0.2558, 0.3923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.3487]\", \"lift_point\": \"[0.1075, 0.3487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.3923]\", \"lift_point\": \"[0.2558, 0.3923]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8747, 0.2714]\", \"lift_point\": \"[0.8747, 0.2714]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5337, 0.1261]\", \"lift_point\": \"[0.5337, 0.1261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.3487]\", \"lift_point\": \"[0.1075, 0.3487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.3923]\", \"lift_point\": \"[0.2558, 0.3923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8747, 0.2714]\", \"lift_point\": \"[0.8747, 0.2714]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.2968]\", \"lift_point\": \"[0.123, 0.2968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.3487]\", \"lift_point\": \"[0.1075, 0.3487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.3923]\", \"lift_point\": \"[0.2558, 0.3923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8747, 0.2714]\", \"lift_point\": \"[0.8747, 0.2714]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9069]\", \"lift_point\": \"[0.0604, 0.9069]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.3487]\", \"lift_point\": \"[0.1075, 0.3487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.3923]\", \"lift_point\": \"[0.2558, 0.3923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8747, 0.2714]\", \"lift_point\": \"[0.8747, 0.2714]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7585, 0.5167]\", \"lift_point\": \"[0.7585, 0.5167]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.2805]\", \"lift_point\": \"[0.0541, 0.2805]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Chile? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.3487]\", \"lift_point\": \"[0.1075, 0.3487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.3923]\", \"lift_point\": \"[0.2558, 0.3923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8747, 0.2714]\", \"lift_point\": \"[0.8747, 0.2714]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7585, 0.5167]\", \"lift_point\": \"[0.7585, 0.5167]\", \"typed_text\": \"\" Goal: What's the news in Chile?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19310_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.1352]\", \"lift_point\": \"[0.543, 0.1352]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.1352]\", \"lift_point\": \"[0.543, 0.1352]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.4126]\", \"lift_point\": \"[0.1093, 0.4126]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.1352]\", \"lift_point\": \"[0.543, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.4126]\", \"lift_point\": \"[0.1093, 0.4126]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.9057]\", \"lift_point\": \"[0.0622, 0.9057]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.1352]\", \"lift_point\": \"[0.543, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.4126]\", \"lift_point\": \"[0.1093, 0.4126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.9057]\", \"lift_point\": \"[0.0622, 0.9057]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0419, 0.2554]\", \"lift_point\": \"[0.0419, 0.2554]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.1352]\", \"lift_point\": \"[0.543, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.4126]\", \"lift_point\": \"[0.1093, 0.4126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.9057]\", \"lift_point\": \"[0.0622, 0.9057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0419, 0.2554]\", \"lift_point\": \"[0.0419, 0.2554]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of sound? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.1409]\", \"lift_point\": \"[0.1368, 0.1409]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.1352]\", \"lift_point\": \"[0.543, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.4126]\", \"lift_point\": \"[0.1093, 0.4126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.9057]\", \"lift_point\": \"[0.0622, 0.9057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0419, 0.2554]\", \"lift_point\": \"[0.0419, 0.2554]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of sound? \" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.3291]\", \"lift_point\": \"[0.1154, 0.3291]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.1352]\", \"lift_point\": \"[0.543, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.4126]\", \"lift_point\": \"[0.1093, 0.4126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0622, 0.9057]\", \"lift_point\": \"[0.0622, 0.9057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0419, 0.2554]\", \"lift_point\": \"[0.0419, 0.2554]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of sound? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.3291]\", \"lift_point\": \"[0.1154, 0.3291]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19321_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.309]\", \"lift_point\": \"[0.1954, 0.309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.309]\", \"lift_point\": \"[0.1954, 0.309]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8997, 0.5628]\", \"lift_point\": \"[0.8997, 0.5628]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.309]\", \"lift_point\": \"[0.1954, 0.309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8997, 0.5628]\", \"lift_point\": \"[0.8997, 0.5628]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9033, 0.642]\", \"lift_point\": \"[0.9033, 0.642]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.309]\", \"lift_point\": \"[0.1954, 0.309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8997, 0.5628]\", \"lift_point\": \"[0.8997, 0.5628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9033, 0.642]\", \"lift_point\": \"[0.9033, 0.642]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0681, 0.1163]\", \"lift_point\": \"[0.0681, 0.1163]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.309]\", \"lift_point\": \"[0.1954, 0.309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8997, 0.5628]\", \"lift_point\": \"[0.8997, 0.5628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9033, 0.642]\", \"lift_point\": \"[0.9033, 0.642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0681, 0.1163]\", \"lift_point\": \"[0.0681, 0.1163]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1405, 0.2047]\", \"lift_point\": \"[0.1405, 0.2047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5677, 0.8324]\", \"lift_point\": \"[0.5677, 0.8324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.309]\", \"lift_point\": \"[0.1954, 0.309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8997, 0.5628]\", \"lift_point\": \"[0.8997, 0.5628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9033, 0.642]\", \"lift_point\": \"[0.9033, 0.642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0681, 0.1163]\", \"lift_point\": \"[0.0681, 0.1163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1405, 0.2047]\", \"lift_point\": \"[0.1405, 0.2047]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19503_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7983, 0.6786]\", \"lift_point\": \"[0.7983, 0.6786]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7983, 0.6786]\", \"lift_point\": \"[0.7983, 0.6786]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6971]\", \"lift_point\": \"[0.7952, 0.6971]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7983, 0.6786]\", \"lift_point\": \"[0.7983, 0.6786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6971]\", \"lift_point\": \"[0.7952, 0.6971]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.107, 0.603]\", \"lift_point\": \"[0.107, 0.603]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7983, 0.6786]\", \"lift_point\": \"[0.7983, 0.6786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6971]\", \"lift_point\": \"[0.7952, 0.6971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.107, 0.603]\", \"lift_point\": \"[0.107, 0.603]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.5384]\", \"lift_point\": \"[0.0724, 0.5384]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7983, 0.6786]\", \"lift_point\": \"[0.7983, 0.6786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6971]\", \"lift_point\": \"[0.7952, 0.6971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.107, 0.603]\", \"lift_point\": \"[0.107, 0.603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.5384]\", \"lift_point\": \"[0.0724, 0.5384]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7983, 0.6786]\", \"lift_point\": \"[0.7983, 0.6786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6971]\", \"lift_point\": \"[0.7952, 0.6971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.107, 0.603]\", \"lift_point\": \"[0.107, 0.603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.5384]\", \"lift_point\": \"[0.0724, 0.5384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.208, 0.5184]\", \"lift_point\": \"[0.208, 0.5184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7894, 0.6932]\", \"lift_point\": \"[0.7894, 0.6932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7983, 0.6786]\", \"lift_point\": \"[0.7983, 0.6786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6971]\", \"lift_point\": \"[0.7952, 0.6971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.107, 0.603]\", \"lift_point\": \"[0.107, 0.603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.5384]\", \"lift_point\": \"[0.0724, 0.5384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.208, 0.5184]\", \"lift_point\": \"[0.208, 0.5184]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19696_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5354]\", \"lift_point\": \"[0.1352, 0.5354]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5354]\", \"lift_point\": \"[0.1352, 0.5354]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5354]\", \"lift_point\": \"[0.1352, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5354]\", \"lift_point\": \"[0.1352, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8087, 0.7014]\", \"lift_point\": \"[0.8087, 0.7014]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5354]\", \"lift_point\": \"[0.1352, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7418, 0.5699]\", \"lift_point\": \"[0.7418, 0.5699]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.4957]\", \"lift_point\": \"[0.0668, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from London to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5249]\", \"lift_point\": \"[0.1352, 0.5249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5354]\", \"lift_point\": \"[0.1352, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7418, 0.5699]\", \"lift_point\": \"[0.7418, 0.5699]\", \"typed_text\": \"\" Goal: Search for flights from London to Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19707_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9171]\", \"lift_point\": \"[0.0703, 0.9171]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9171]\", \"lift_point\": \"[0.0703, 0.9171]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9931, 0.7978]\", \"lift_point\": \"[0.9931, 0.7978]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9171]\", \"lift_point\": \"[0.0703, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9931, 0.7978]\", \"lift_point\": \"[0.9931, 0.7978]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9171]\", \"lift_point\": \"[0.0703, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9931, 0.7978]\", \"lift_point\": \"[0.9931, 0.7978]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Chicago\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1514, 0.338]\", \"lift_point\": \"[0.1514, 0.338]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.6755]\", \"lift_point\": \"[0.5107, 0.6755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9171]\", \"lift_point\": \"[0.0703, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9931, 0.7978]\", \"lift_point\": \"[0.9931, 0.7978]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1514, 0.338]\", \"lift_point\": \"[0.1514, 0.338]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.4203]\", \"lift_point\": \"[0.3338, 0.4203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.5856]\", \"lift_point\": \"[0.1326, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.6506]\", \"lift_point\": \"[0.1333, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.126, 0.5796]\", \"lift_point\": \"[0.126, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9171]\", \"lift_point\": \"[0.0703, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9931, 0.7978]\", \"lift_point\": \"[0.9931, 0.7978]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1514, 0.338]\", \"lift_point\": \"[0.1514, 0.338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.4203]\", \"lift_point\": \"[0.3338, 0.4203]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19765_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3751, 0.4752]\", \"lift_point\": \"[0.3751, 0.4752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2126, 0.6494]\", \"lift_point\": \"[0.2126, 0.6494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4797]\", \"lift_point\": \"[0.1319, 0.4797]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1314, 0.9394]\", \"lift_point\": \"[0.1314, 0.9394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4797]\", \"lift_point\": \"[0.1319, 0.4797]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.4727]\", \"lift_point\": \"[0.1261, 0.4727]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4797]\", \"lift_point\": \"[0.1319, 0.4797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.4727]\", \"lift_point\": \"[0.1261, 0.4727]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6105, 0.4638]\", \"lift_point\": \"[0.6105, 0.4638]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4797]\", \"lift_point\": \"[0.1319, 0.4797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.4727]\", \"lift_point\": \"[0.1261, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6105, 0.4638]\", \"lift_point\": \"[0.6105, 0.4638]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19820_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.198, 0.6297]\", \"lift_point\": \"[0.198, 0.6297]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.198, 0.6297]\", \"lift_point\": \"[0.198, 0.6297]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.9271]\", \"lift_point\": \"[0.1238, 0.9271]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.198, 0.6297]\", \"lift_point\": \"[0.198, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.9271]\", \"lift_point\": \"[0.1238, 0.9271]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.198, 0.6297]\", \"lift_point\": \"[0.198, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.9271]\", \"lift_point\": \"[0.1238, 0.9271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.198, 0.6297]\", \"lift_point\": \"[0.198, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.9271]\", \"lift_point\": \"[0.1238, 0.9271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2938, 0.2877]\", \"lift_point\": \"[0.2938, 0.2877]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.198, 0.6297]\", \"lift_point\": \"[0.198, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.9271]\", \"lift_point\": \"[0.1238, 0.9271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2938, 0.2877]\", \"lift_point\": \"[0.2938, 0.2877]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.4167]\", \"lift_point\": \"[0.6039, 0.4167]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3666, 0.4903]\", \"lift_point\": \"[0.3666, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.198, 0.6297]\", \"lift_point\": \"[0.198, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.9271]\", \"lift_point\": \"[0.1238, 0.9271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2938, 0.2877]\", \"lift_point\": \"[0.2938, 0.2877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.4167]\", \"lift_point\": \"[0.6039, 0.4167]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19849_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3669, 0.4972]\", \"lift_point\": \"[0.3669, 0.4972]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3669, 0.4972]\", \"lift_point\": \"[0.3669, 0.4972]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5386]\", \"lift_point\": \"[0.0589, 0.5386]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3669, 0.4972]\", \"lift_point\": \"[0.3669, 0.4972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5386]\", \"lift_point\": \"[0.0589, 0.5386]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.9062]\", \"lift_point\": \"[0.0668, 0.9062]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3669, 0.4972]\", \"lift_point\": \"[0.3669, 0.4972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5386]\", \"lift_point\": \"[0.0589, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.9062]\", \"lift_point\": \"[0.0668, 0.9062]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a jet?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3669, 0.4972]\", \"lift_point\": \"[0.3669, 0.4972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5386]\", \"lift_point\": \"[0.0589, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.9062]\", \"lift_point\": \"[0.0668, 0.9062]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a jet?\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.5866]\", \"lift_point\": \"[0.1297, 0.5866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3669, 0.4972]\", \"lift_point\": \"[0.3669, 0.4972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5386]\", \"lift_point\": \"[0.0589, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0668, 0.9062]\", \"lift_point\": \"[0.0668, 0.9062]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a jet?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.5866]\", \"lift_point\": \"[0.1297, 0.5866]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/19987_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20116_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3591, 0.516]\", \"lift_point\": \"[0.3591, 0.516]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20116_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3591, 0.516]\", \"lift_point\": \"[0.3591, 0.516]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5116]\", \"lift_point\": \"[0.3725, 0.5116]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20116_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3591, 0.516]\", \"lift_point\": \"[0.3591, 0.516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5116]\", \"lift_point\": \"[0.3725, 0.5116]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20116_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3373, 0.5155]\", \"lift_point\": \"[0.3373, 0.5155]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3373, 0.5155]\", \"lift_point\": \"[0.3373, 0.5155]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0699, 0.4292]\", \"lift_point\": \"[0.0699, 0.4292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3373, 0.5155]\", \"lift_point\": \"[0.3373, 0.5155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0699, 0.4292]\", \"lift_point\": \"[0.0699, 0.4292]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9155]\", \"lift_point\": \"[0.0711, 0.9155]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3373, 0.5155]\", \"lift_point\": \"[0.3373, 0.5155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0699, 0.4292]\", \"lift_point\": \"[0.0699, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9155]\", \"lift_point\": \"[0.0711, 0.9155]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3373, 0.5155]\", \"lift_point\": \"[0.3373, 0.5155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0699, 0.4292]\", \"lift_point\": \"[0.0699, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9155]\", \"lift_point\": \"[0.0711, 0.9155]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5079]\", \"lift_point\": \"[0.1339, 0.5079]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3373, 0.5155]\", \"lift_point\": \"[0.3373, 0.5155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0699, 0.4292]\", \"lift_point\": \"[0.0699, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9155]\", \"lift_point\": \"[0.0711, 0.9155]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5079]\", \"lift_point\": \"[0.1339, 0.5079]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20212_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9425]\", \"lift_point\": \"[0.0638, 0.9425]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9425]\", \"lift_point\": \"[0.0638, 0.9425]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.8502]\", \"lift_point\": \"[0.0572, 0.8502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9425]\", \"lift_point\": \"[0.0638, 0.9425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.8502]\", \"lift_point\": \"[0.0572, 0.8502]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.909]\", \"lift_point\": \"[0.1329, 0.909]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9425]\", \"lift_point\": \"[0.0638, 0.9425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.8502]\", \"lift_point\": \"[0.0572, 0.8502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.909]\", \"lift_point\": \"[0.1329, 0.909]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.0469]\", \"lift_point\": \"[0.0513, 0.0469]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9425]\", \"lift_point\": \"[0.0638, 0.9425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.8502]\", \"lift_point\": \"[0.0572, 0.8502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.909]\", \"lift_point\": \"[0.1329, 0.909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.0469]\", \"lift_point\": \"[0.0513, 0.0469]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.9125]\", \"lift_point\": \"[0.6324, 0.9125]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9425]\", \"lift_point\": \"[0.0638, 0.9425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.8502]\", \"lift_point\": \"[0.0572, 0.8502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.909]\", \"lift_point\": \"[0.1329, 0.909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.0469]\", \"lift_point\": \"[0.0513, 0.0469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.9125]\", \"lift_point\": \"[0.6324, 0.9125]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3924, 0.5924]\", \"lift_point\": \"[0.3924, 0.5924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.508]\", \"lift_point\": \"[0.3575, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.9425]\", \"lift_point\": \"[0.0638, 0.9425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.8502]\", \"lift_point\": \"[0.0572, 0.8502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.909]\", \"lift_point\": \"[0.1329, 0.909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.0469]\", \"lift_point\": \"[0.0513, 0.0469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6324, 0.9125]\", \"lift_point\": \"[0.6324, 0.9125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3924, 0.5924]\", \"lift_point\": \"[0.3924, 0.5924]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20436_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.4842]\", \"lift_point\": \"[0.3643, 0.4842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.468]\", \"lift_point\": \"[0.065, 0.468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4356]\", \"lift_point\": \"[0.1414, 0.4356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6042, 0.2889]\", \"lift_point\": \"[0.6042, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2661, 0.5474]\", \"lift_point\": \"[0.2661, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.282, 0.3245]\", \"lift_point\": \"[0.282, 0.3245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4006, 0.3353]\", \"lift_point\": \"[0.4006, 0.3353]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.0765]\", \"lift_point\": \"[0.1241, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.0705]\", \"lift_point\": \"[0.1278, 0.0705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.1043]\", \"lift_point\": \"[0.1288, 0.1043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3287, 0.2535]\", \"lift_point\": \"[0.3287, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5125, 0.3515]\", \"lift_point\": \"[0.5125, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9362, 0.3203]\", \"lift_point\": \"[0.9362, 0.3203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9362, 0.3203]\", \"lift_point\": \"[0.9362, 0.3203]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9362, 0.3203]\", \"lift_point\": \"[0.9362, 0.3203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7351, 0.4954]\", \"lift_point\": \"[0.7351, 0.4954]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9362, 0.3203]\", \"lift_point\": \"[0.9362, 0.3203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7351, 0.4954]\", \"lift_point\": \"[0.7351, 0.4954]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20520_47.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of light?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3718, 0.5032]\", \"lift_point\": \"[0.3718, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3641, 0.5107]\", \"lift_point\": \"[0.3641, 0.5107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0365, 0.5117]\", \"lift_point\": \"[0.0365, 0.5117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0496, 0.5092]\", \"lift_point\": \"[0.0496, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2097, 0.3362]\", \"lift_point\": \"[0.2097, 0.3362]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6377, 0.4206]\", \"lift_point\": \"[0.6377, 0.4206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4822, 0.5822]\", \"lift_point\": \"[0.4822, 0.5822]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4822, 0.5822]\", \"lift_point\": \"[0.4822, 0.5822]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20581_38.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3558, 0.4939]\", \"lift_point\": \"[0.3558, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.5279]\", \"lift_point\": \"[0.0474, 0.5279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2998, 0.3466]\", \"lift_point\": \"[0.2998, 0.3466]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2998, 0.3466]\", \"lift_point\": \"[0.2998, 0.3466]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20596_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3745, 0.4935]\", \"lift_point\": \"[0.3745, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.4256]\", \"lift_point\": \"[0.0542, 0.4256]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.6541]\", \"lift_point\": \"[0.3063, 0.6541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3338, 0.5289]\", \"lift_point\": \"[0.3338, 0.5289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1764, 0.6049]\", \"lift_point\": \"[0.1764, 0.6049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4134, 0.6631]\", \"lift_point\": \"[0.4134, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2562, 0.9481]\", \"lift_point\": \"[0.2562, 0.9481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.263, 0.7532]\", \"lift_point\": \"[0.263, 0.7532]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.263, 0.7532]\", \"lift_point\": \"[0.263, 0.7532]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.9582]\", \"lift_point\": \"[0.1406, 0.9582]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.263, 0.7532]\", \"lift_point\": \"[0.263, 0.7532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.9582]\", \"lift_point\": \"[0.1406, 0.9582]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20704_54.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.6666]\", \"lift_point\": \"[0.5045, 0.6666]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.6666]\", \"lift_point\": \"[0.5045, 0.6666]\", \"typed_text\": \"\" Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.6666]\", \"lift_point\": \"[0.5045, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\" Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2155, 0.1766]\", \"lift_point\": \"[0.2155, 0.1766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.6666]\", \"lift_point\": \"[0.5045, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2155, 0.1766]\", \"lift_point\": \"[0.2155, 0.1766]\", \"typed_text\": \"\" Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.6666]\", \"lift_point\": \"[0.5045, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2155, 0.1766]\", \"lift_point\": \"[0.2155, 0.1766]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\" Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1415, 0.5113]\", \"lift_point\": \"[0.1415, 0.5113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.6666]\", \"lift_point\": \"[0.5045, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2155, 0.1766]\", \"lift_point\": \"[0.2155, 0.1766]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Los Angeles?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1415, 0.5113]\", \"lift_point\": \"[0.1415, 0.5113]\", \"typed_text\": \"\" Goal: What's a good restaurant in Los Angeles?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20890_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4965, 0.7021]\", \"lift_point\": \"[0.4965, 0.7021]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4965, 0.7021]\", \"lift_point\": \"[0.4965, 0.7021]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.6026]\", \"lift_point\": \"[0.1406, 0.6026]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4965, 0.7021]\", \"lift_point\": \"[0.4965, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.6026]\", \"lift_point\": \"[0.1406, 0.6026]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.8851]\", \"lift_point\": \"[0.0632, 0.8851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4965, 0.7021]\", \"lift_point\": \"[0.4965, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.6026]\", \"lift_point\": \"[0.1406, 0.6026]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.8851]\", \"lift_point\": \"[0.0632, 0.8851]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4965, 0.7021]\", \"lift_point\": \"[0.4965, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.6026]\", \"lift_point\": \"[0.1406, 0.6026]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.8851]\", \"lift_point\": \"[0.0632, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1556, 0.3143]\", \"lift_point\": \"[0.1556, 0.3143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4965, 0.7021]\", \"lift_point\": \"[0.4965, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.6026]\", \"lift_point\": \"[0.1406, 0.6026]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.8851]\", \"lift_point\": \"[0.0632, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1556, 0.3143]\", \"lift_point\": \"[0.1556, 0.3143]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/20912_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.6206]\", \"lift_point\": \"[0.3479, 0.6206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.6206]\", \"lift_point\": \"[0.3479, 0.6206]\", \"typed_text\": \"\" Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.3953]\", \"lift_point\": \"[0.07, 0.3953]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.6206]\", \"lift_point\": \"[0.3479, 0.6206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.3953]\", \"lift_point\": \"[0.07, 0.3953]\", \"typed_text\": \"\" Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Paraguay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.6206]\", \"lift_point\": \"[0.3479, 0.6206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.3953]\", \"lift_point\": \"[0.07, 0.3953]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Paraguay?\" Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.6206]\", \"lift_point\": \"[0.3479, 0.6206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.3953]\", \"lift_point\": \"[0.07, 0.3953]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Paraguay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3516, 0.3604]\", \"lift_point\": \"[0.3516, 0.3604]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.6206]\", \"lift_point\": \"[0.3479, 0.6206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.3953]\", \"lift_point\": \"[0.07, 0.3953]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Paraguay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3516, 0.3604]\", \"lift_point\": \"[0.3516, 0.3604]\", \"typed_text\": \"\" Goal: What's the news in Paraguay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21075_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21098_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3542, 0.5629]\", \"lift_point\": \"[0.3542, 0.5629]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21098_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3542, 0.5629]\", \"lift_point\": \"[0.3542, 0.5629]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.4942]\", \"lift_point\": \"[0.0555, 0.4942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21098_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3542, 0.5629]\", \"lift_point\": \"[0.3542, 0.5629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.4942]\", \"lift_point\": \"[0.0555, 0.4942]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Bolivia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21098_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3542, 0.5629]\", \"lift_point\": \"[0.3542, 0.5629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.4942]\", \"lift_point\": \"[0.0555, 0.4942]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Bolivia\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21098_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3542, 0.5629]\", \"lift_point\": \"[0.3542, 0.5629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0555, 0.4942]\", \"lift_point\": \"[0.0555, 0.4942]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Bolivia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Bolivia>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21098_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.7016]\", \"lift_point\": \"[0.7689, 0.7016]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21162_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.7016]\", \"lift_point\": \"[0.7689, 0.7016]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.4626]\", \"lift_point\": \"[0.885, 0.4626]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21162_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.7016]\", \"lift_point\": \"[0.7689, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.4626]\", \"lift_point\": \"[0.885, 0.4626]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8937, 0.8222]\", \"lift_point\": \"[0.8937, 0.8222]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21162_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.7016]\", \"lift_point\": \"[0.7689, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.4626]\", \"lift_point\": \"[0.885, 0.4626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8937, 0.8222]\", \"lift_point\": \"[0.8937, 0.8222]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3026, 0.6258]\", \"lift_point\": \"[0.3026, 0.6258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21162_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.7016]\", \"lift_point\": \"[0.7689, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.4626]\", \"lift_point\": \"[0.885, 0.4626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8937, 0.8222]\", \"lift_point\": \"[0.8937, 0.8222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3026, 0.6258]\", \"lift_point\": \"[0.3026, 0.6258]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Switzerland\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21162_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.7016]\", \"lift_point\": \"[0.7689, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.4626]\", \"lift_point\": \"[0.885, 0.4626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8937, 0.8222]\", \"lift_point\": \"[0.8937, 0.8222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3026, 0.6258]\", \"lift_point\": \"[0.3026, 0.6258]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Switzerland\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.3284]\", \"lift_point\": \"[0.1308, 0.3284]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21162_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7689, 0.7016]\", \"lift_point\": \"[0.7689, 0.7016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.4626]\", \"lift_point\": \"[0.885, 0.4626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8937, 0.8222]\", \"lift_point\": \"[0.8937, 0.8222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3026, 0.6258]\", \"lift_point\": \"[0.3026, 0.6258]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Switzerland\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.3284]\", \"lift_point\": \"[0.1308, 0.3284]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21162_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.492]\", \"lift_point\": \"[0.7836, 0.492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.8519]\", \"lift_point\": \"[0.0604, 0.8519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2892, 0.4823]\", \"lift_point\": \"[0.2892, 0.4823]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.2291]\", \"lift_point\": \"[0.1179, 0.2291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2892, 0.4823]\", \"lift_point\": \"[0.2892, 0.4823]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2258, 0.8914]\", \"lift_point\": \"[0.2258, 0.8914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2892, 0.4823]\", \"lift_point\": \"[0.2892, 0.4823]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2258, 0.8914]\", \"lift_point\": \"[0.2258, 0.8914]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21290_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21316_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6882]\", \"lift_point\": \"[0.7794, 0.6882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21316_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6882]\", \"lift_point\": \"[0.7794, 0.6882]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.5767]\", \"lift_point\": \"[0.0602, 0.5767]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21316_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6882]\", \"lift_point\": \"[0.7794, 0.6882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.5767]\", \"lift_point\": \"[0.0602, 0.5767]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotels in Miami\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21316_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6882]\", \"lift_point\": \"[0.7794, 0.6882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.5767]\", \"lift_point\": \"[0.0602, 0.5767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotels in Miami\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1218, 0.322]\", \"lift_point\": \"[0.1218, 0.322]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21316_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6882]\", \"lift_point\": \"[0.7794, 0.6882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0602, 0.5767]\", \"lift_point\": \"[0.0602, 0.5767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotels in Miami\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1218, 0.322]\", \"lift_point\": \"[0.1218, 0.322]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21316_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0993, 0.0948]\", \"lift_point\": \"[0.0993, 0.0948]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0993, 0.0948]\", \"lift_point\": \"[0.0993, 0.0948]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.376]\", \"lift_point\": \"[0.1063, 0.376]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0993, 0.0948]\", \"lift_point\": \"[0.0993, 0.0948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.376]\", \"lift_point\": \"[0.1063, 0.376]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"eBay \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0993, 0.0948]\", \"lift_point\": \"[0.0993, 0.0948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.376]\", \"lift_point\": \"[0.1063, 0.376]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"eBay \" Goal: Install the eBay app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0993, 0.0948]\", \"lift_point\": \"[0.0993, 0.0948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.376]\", \"lift_point\": \"[0.1063, 0.376]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"eBay \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.3718]\", \"lift_point\": \"[0.2193, 0.3718]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0993, 0.0948]\", \"lift_point\": \"[0.0993, 0.0948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.376]\", \"lift_point\": \"[0.1063, 0.376]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"eBay \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.3718]\", \"lift_point\": \"[0.2193, 0.3718]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2617, 0.823]\", \"lift_point\": \"[0.2617, 0.823]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.888, 0.8691]\", \"lift_point\": \"[0.888, 0.8691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0993, 0.0948]\", \"lift_point\": \"[0.0993, 0.0948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1063, 0.376]\", \"lift_point\": \"[0.1063, 0.376]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"eBay \" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.3718]\", \"lift_point\": \"[0.2193, 0.3718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2617, 0.823]\", \"lift_point\": \"[0.2617, 0.823]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21601_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2066, 0.4316]\", \"lift_point\": \"[0.2066, 0.4316]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2066, 0.4316]\", \"lift_point\": \"[0.2066, 0.4316]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2066, 0.4316]\", \"lift_point\": \"[0.2066, 0.4316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4929, 0.4908]\", \"lift_point\": \"[0.4929, 0.4908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2066, 0.4316]\", \"lift_point\": \"[0.2066, 0.4316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5093]\", \"lift_point\": \"[0.8404, 0.5093]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3479, 0.5891]\", \"lift_point\": \"[0.3479, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2221, 0.8792]\", \"lift_point\": \"[0.2221, 0.8792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2066, 0.4316]\", \"lift_point\": \"[0.2066, 0.4316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5093]\", \"lift_point\": \"[0.8404, 0.5093]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21829_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21907_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21907_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21907_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7072, 0.1192]\", \"lift_point\": \"[0.7072, 0.1192]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21907_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7072, 0.1192]\", \"lift_point\": \"[0.7072, 0.1192]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21907_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3325, 0.5901]\", \"lift_point\": \"[0.3325, 0.5901]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3325, 0.5901]\", \"lift_point\": \"[0.3325, 0.5901]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2106, 0.8731]\", \"lift_point\": \"[0.2106, 0.8731]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.515]\", \"lift_point\": \"[0.1053, 0.515]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3325, 0.5901]\", \"lift_point\": \"[0.3325, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2106, 0.8731]\", \"lift_point\": \"[0.2106, 0.8731]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant near me?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3325, 0.5901]\", \"lift_point\": \"[0.3325, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2106, 0.8731]\", \"lift_point\": \"[0.2106, 0.8731]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant near me?\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3325, 0.5901]\", \"lift_point\": \"[0.3325, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2106, 0.8731]\", \"lift_point\": \"[0.2106, 0.8731]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant near me?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8743, 0.5131]\", \"lift_point\": \"[0.8743, 0.5131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5031, 0.4783]\", \"lift_point\": \"[0.5031, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3325, 0.5901]\", \"lift_point\": \"[0.3325, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2106, 0.8731]\", \"lift_point\": \"[0.2106, 0.8731]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant near me?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8743, 0.5131]\", \"lift_point\": \"[0.8743, 0.5131]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/21940_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5405, 0.1408]\", \"lift_point\": \"[0.5405, 0.1408]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22042_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5405, 0.1408]\", \"lift_point\": \"[0.5405, 0.1408]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4514]\", \"lift_point\": \"[0.0985, 0.4514]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22042_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5405, 0.1408]\", \"lift_point\": \"[0.5405, 0.1408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4514]\", \"lift_point\": \"[0.0985, 0.4514]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronic store\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22042_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5405, 0.1408]\", \"lift_point\": \"[0.5405, 0.1408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4514]\", \"lift_point\": \"[0.0985, 0.4514]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronic store\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2042, 0.4117]\", \"lift_point\": \"[0.2042, 0.4117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22042_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5405, 0.1408]\", \"lift_point\": \"[0.5405, 0.1408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4514]\", \"lift_point\": \"[0.0985, 0.4514]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronic store\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2042, 0.4117]\", \"lift_point\": \"[0.2042, 0.4117]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7959, 0.9214]\", \"lift_point\": \"[0.7959, 0.9214]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22042_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5405, 0.1408]\", \"lift_point\": \"[0.5405, 0.1408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4514]\", \"lift_point\": \"[0.0985, 0.4514]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronic store\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2042, 0.4117]\", \"lift_point\": \"[0.2042, 0.4117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7959, 0.9214]\", \"lift_point\": \"[0.7959, 0.9214]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22042_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1848, 0.5555]\", \"lift_point\": \"[0.1848, 0.5555]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1848, 0.5555]\", \"lift_point\": \"[0.1848, 0.5555]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5601, 0.371]\", \"lift_point\": \"[0.5601, 0.371]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1848, 0.5555]\", \"lift_point\": \"[0.1848, 0.5555]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5601, 0.371]\", \"lift_point\": \"[0.5601, 0.371]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5774, 0.7847]\", \"lift_point\": \"[0.5774, 0.7847]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1848, 0.5555]\", \"lift_point\": \"[0.1848, 0.5555]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5601, 0.371]\", \"lift_point\": \"[0.5601, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5774, 0.7847]\", \"lift_point\": \"[0.5774, 0.7847]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1848, 0.5555]\", \"lift_point\": \"[0.1848, 0.5555]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5601, 0.371]\", \"lift_point\": \"[0.5601, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5774, 0.7847]\", \"lift_point\": \"[0.5774, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3079, 0.9607]\", \"lift_point\": \"[0.3079, 0.9607]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7234, 0.6351]\", \"lift_point\": \"[0.7234, 0.6351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4722]\", \"lift_point\": \"[0.1028, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at McDonalds\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1848, 0.5555]\", \"lift_point\": \"[0.1848, 0.5555]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5601, 0.371]\", \"lift_point\": \"[0.5601, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5774, 0.7847]\", \"lift_point\": \"[0.5774, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3079, 0.9607]\", \"lift_point\": \"[0.3079, 0.9607]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22058_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7349, 0.6118]\", \"lift_point\": \"[0.7349, 0.6118]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22092_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7349, 0.6118]\", \"lift_point\": \"[0.7349, 0.6118]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22092_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7349, 0.6118]\", \"lift_point\": \"[0.7349, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7081, 0.6262]\", \"lift_point\": \"[0.7081, 0.6262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22092_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7349, 0.6118]\", \"lift_point\": \"[0.7349, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7081, 0.6262]\", \"lift_point\": \"[0.7081, 0.6262]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0856, 0.4623]\", \"lift_point\": \"[0.0856, 0.4623]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22092_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7349, 0.6118]\", \"lift_point\": \"[0.7349, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7081, 0.6262]\", \"lift_point\": \"[0.7081, 0.6262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0856, 0.4623]\", \"lift_point\": \"[0.0856, 0.4623]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden ?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22092_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7349, 0.6118]\", \"lift_point\": \"[0.7349, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7081, 0.6262]\", \"lift_point\": \"[0.7081, 0.6262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0856, 0.4623]\", \"lift_point\": \"[0.0856, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden?\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2007, 0.6839]\", \"lift_point\": \"[0.2007, 0.6839]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22092_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7349, 0.6118]\", \"lift_point\": \"[0.7349, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7081, 0.6262]\", \"lift_point\": \"[0.7081, 0.6262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0856, 0.4623]\", \"lift_point\": \"[0.0856, 0.4623]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of sweden?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2007, 0.6839]\", \"lift_point\": \"[0.2007, 0.6839]\", \"typed_text\": \"\" Goal: What is the capital of Sweden?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22092_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5232, 0.1533]\", \"lift_point\": \"[0.5232, 0.1533]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22123_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5232, 0.1533]\", \"lift_point\": \"[0.5232, 0.1533]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0991, 0.6281]\", \"lift_point\": \"[0.0991, 0.6281]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22123_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5232, 0.1533]\", \"lift_point\": \"[0.5232, 0.1533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0991, 0.6281]\", \"lift_point\": \"[0.0991, 0.6281]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0909, 0.9027]\", \"lift_point\": \"[0.0909, 0.9027]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22123_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5232, 0.1533]\", \"lift_point\": \"[0.5232, 0.1533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0991, 0.6281]\", \"lift_point\": \"[0.0991, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0909, 0.9027]\", \"lift_point\": \"[0.0909, 0.9027]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Diego\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22123_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5232, 0.1533]\", \"lift_point\": \"[0.5232, 0.1533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0991, 0.6281]\", \"lift_point\": \"[0.0991, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0909, 0.9027]\", \"lift_point\": \"[0.0909, 0.9027]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Diego\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2215, 0.5013]\", \"lift_point\": \"[0.2215, 0.5013]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22123_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5232, 0.1533]\", \"lift_point\": \"[0.5232, 0.1533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0991, 0.6281]\", \"lift_point\": \"[0.0991, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0909, 0.9027]\", \"lift_point\": \"[0.0909, 0.9027]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2215, 0.5013]\", \"lift_point\": \"[0.2215, 0.5013]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22123_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22128_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7163, 0.5936]\", \"lift_point\": \"[0.7163, 0.5936]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22128_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7163, 0.5936]\", \"lift_point\": \"[0.7163, 0.5936]\", \"typed_text\": \"\" Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0967, 0.9101]\", \"lift_point\": \"[0.0967, 0.9101]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22128_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7163, 0.5936]\", \"lift_point\": \"[0.7163, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0967, 0.9101]\", \"lift_point\": \"[0.0967, 0.9101]\", \"typed_text\": \"\" Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2619, 0.6533]\", \"lift_point\": \"[0.2619, 0.6533]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22128_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7163, 0.5936]\", \"lift_point\": \"[0.7163, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0967, 0.9101]\", \"lift_point\": \"[0.0967, 0.9101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2619, 0.6533]\", \"lift_point\": \"[0.2619, 0.6533]\", \"typed_text\": \"\" Goal: Open a new Chrome private window>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22128_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22302_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7279, 0.6578]\", \"lift_point\": \"[0.7279, 0.6578]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22302_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7279, 0.6578]\", \"lift_point\": \"[0.7279, 0.6578]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4437, 0.5139]\", \"lift_point\": \"[0.4437, 0.5139]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22302_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7279, 0.6578]\", \"lift_point\": \"[0.7279, 0.6578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4437, 0.5139]\", \"lift_point\": \"[0.4437, 0.5139]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from NYC to London\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22302_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7279, 0.6578]\", \"lift_point\": \"[0.7279, 0.6578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4437, 0.5139]\", \"lift_point\": \"[0.4437, 0.5139]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from NYC to London\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2149, 0.6706]\", \"lift_point\": \"[0.2149, 0.6706]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22302_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7279, 0.6578]\", \"lift_point\": \"[0.7279, 0.6578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4437, 0.5139]\", \"lift_point\": \"[0.4437, 0.5139]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from NYC to London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2149, 0.6706]\", \"lift_point\": \"[0.2149, 0.6706]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22302_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2494, 0.168]\", \"lift_point\": \"[0.2494, 0.168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22350_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2494, 0.168]\", \"lift_point\": \"[0.2494, 0.168]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5252, 0.2376]\", \"lift_point\": \"[0.5252, 0.2376]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22350_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2494, 0.168]\", \"lift_point\": \"[0.2494, 0.168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5252, 0.2376]\", \"lift_point\": \"[0.5252, 0.2376]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2077, 0.787]\", \"lift_point\": \"[0.2077, 0.787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22350_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2494, 0.168]\", \"lift_point\": \"[0.2494, 0.168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5252, 0.2376]\", \"lift_point\": \"[0.5252, 0.2376]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2077, 0.787]\", \"lift_point\": \"[0.2077, 0.787]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7674, 0.8042]\", \"lift_point\": \"[0.7674, 0.8042]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22350_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2494, 0.168]\", \"lift_point\": \"[0.2494, 0.168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5252, 0.2376]\", \"lift_point\": \"[0.5252, 0.2376]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2077, 0.787]\", \"lift_point\": \"[0.2077, 0.787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7674, 0.8042]\", \"lift_point\": \"[0.7674, 0.8042]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22350_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the Vizio TV cost?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22356_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the Vizio TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.6311]\", \"lift_point\": \"[0.7345, 0.6311]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22356_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.6311]\", \"lift_point\": \"[0.7345, 0.6311]\", \"typed_text\": \"\" Goal: How much does the Vizio TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1033, 0.4153]\", \"lift_point\": \"[0.1033, 0.4153]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22356_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.6311]\", \"lift_point\": \"[0.7345, 0.6311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1033, 0.4153]\", \"lift_point\": \"[0.1033, 0.4153]\", \"typed_text\": \"\" Goal: How much does the Vizio TV cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the Vizio TV cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22356_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.6311]\", \"lift_point\": \"[0.7345, 0.6311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1033, 0.4153]\", \"lift_point\": \"[0.1033, 0.4153]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the Vizio TV cost?\" Goal: How much does the Vizio TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1992, 0.4675]\", \"lift_point\": \"[0.1992, 0.4675]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22356_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.6311]\", \"lift_point\": \"[0.7345, 0.6311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1033, 0.4153]\", \"lift_point\": \"[0.1033, 0.4153]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the Vizio TV cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1992, 0.4675]\", \"lift_point\": \"[0.1992, 0.4675]\", \"typed_text\": \"\" Goal: How much does the Vizio TV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22356_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in London>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22393_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5714, 0.1792]\", \"lift_point\": \"[0.5714, 0.1792]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22393_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5714, 0.1792]\", \"lift_point\": \"[0.5714, 0.1792]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1141, 0.5455]\", \"lift_point\": \"[0.1141, 0.5455]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22393_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5714, 0.1792]\", \"lift_point\": \"[0.5714, 0.1792]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1141, 0.5455]\", \"lift_point\": \"[0.1141, 0.5455]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0956, 0.902]\", \"lift_point\": \"[0.0956, 0.902]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22393_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5714, 0.1792]\", \"lift_point\": \"[0.5714, 0.1792]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1141, 0.5455]\", \"lift_point\": \"[0.1141, 0.5455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0956, 0.902]\", \"lift_point\": \"[0.0956, 0.902]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in london\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22393_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5714, 0.1792]\", \"lift_point\": \"[0.5714, 0.1792]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1141, 0.5455]\", \"lift_point\": \"[0.1141, 0.5455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0956, 0.902]\", \"lift_point\": \"[0.0956, 0.902]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in london\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.277, 0.3544]\", \"lift_point\": \"[0.277, 0.3544]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22393_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5714, 0.1792]\", \"lift_point\": \"[0.5714, 0.1792]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1141, 0.5455]\", \"lift_point\": \"[0.1141, 0.5455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0956, 0.902]\", \"lift_point\": \"[0.0956, 0.902]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in london\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.277, 0.3544]\", \"lift_point\": \"[0.277, 0.3544]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22393_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7319, 0.6021]\", \"lift_point\": \"[0.7319, 0.6021]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7319, 0.6021]\", \"lift_point\": \"[0.7319, 0.6021]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0853, 0.3733]\", \"lift_point\": \"[0.0853, 0.3733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7319, 0.6021]\", \"lift_point\": \"[0.7319, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0853, 0.3733]\", \"lift_point\": \"[0.0853, 0.3733]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Buenos aires to Tokyo\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7319, 0.6021]\", \"lift_point\": \"[0.7319, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0853, 0.3733]\", \"lift_point\": \"[0.0853, 0.3733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Buenos aires to Tokyo\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.67]\", \"lift_point\": \"[0.0817, 0.67]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7319, 0.6021]\", \"lift_point\": \"[0.7319, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0853, 0.3733]\", \"lift_point\": \"[0.0853, 0.3733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Buenos aires to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.67]\", \"lift_point\": \"[0.0817, 0.67]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.4818]\", \"lift_point\": \"[0.1806, 0.4818]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5653, 0.1722]\", \"lift_point\": \"[0.5653, 0.1722]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7319, 0.6021]\", \"lift_point\": \"[0.7319, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0853, 0.3733]\", \"lift_point\": \"[0.0853, 0.3733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Buenos aires to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.67]\", \"lift_point\": \"[0.0817, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.4818]\", \"lift_point\": \"[0.1806, 0.4818]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Tokyo>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22402_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2142, 0.6479]\", \"lift_point\": \"[0.2142, 0.6479]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2142, 0.6479]\", \"lift_point\": \"[0.2142, 0.6479]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2142, 0.6479]\", \"lift_point\": \"[0.2142, 0.6479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2142, 0.6479]\", \"lift_point\": \"[0.2142, 0.6479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2142, 0.6479]\", \"lift_point\": \"[0.2142, 0.6479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.741, 0.5892]\", \"lift_point\": \"[0.741, 0.5892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0799, 0.5852]\", \"lift_point\": \"[0.0799, 0.5852]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2142, 0.6479]\", \"lift_point\": \"[0.2142, 0.6479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5744, 0.4222]\", \"lift_point\": \"[0.5744, 0.4222]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22412_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22425_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5056, 0.673]\", \"lift_point\": \"[0.5056, 0.673]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22425_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5056, 0.673]\", \"lift_point\": \"[0.5056, 0.673]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2062, 0.545]\", \"lift_point\": \"[0.2062, 0.545]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22425_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5056, 0.673]\", \"lift_point\": \"[0.5056, 0.673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2062, 0.545]\", \"lift_point\": \"[0.2062, 0.545]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22425_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5056, 0.673]\", \"lift_point\": \"[0.5056, 0.673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2062, 0.545]\", \"lift_point\": \"[0.2062, 0.545]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3907]\", \"lift_point\": \"[0.123, 0.3907]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22425_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5056, 0.673]\", \"lift_point\": \"[0.5056, 0.673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2062, 0.545]\", \"lift_point\": \"[0.2062, 0.545]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.123, 0.3907]\", \"lift_point\": \"[0.123, 0.3907]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22425_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8121, 0.7133]\", \"lift_point\": \"[0.8121, 0.7133]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22459_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8121, 0.7133]\", \"lift_point\": \"[0.8121, 0.7133]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9398]\", \"lift_point\": \"[0.0563, 0.9398]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22459_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8121, 0.7133]\", \"lift_point\": \"[0.8121, 0.7133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9398]\", \"lift_point\": \"[0.0563, 0.9398]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1614, 0.6418]\", \"lift_point\": \"[0.1614, 0.6418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22459_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8121, 0.7133]\", \"lift_point\": \"[0.8121, 0.7133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.9398]\", \"lift_point\": \"[0.0563, 0.9398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1614, 0.6418]\", \"lift_point\": \"[0.1614, 0.6418]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22459_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5025, 0.8773]\", \"lift_point\": \"[0.5025, 0.8773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5025, 0.8773]\", \"lift_point\": \"[0.5025, 0.8773]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.9506]\", \"lift_point\": \"[0.0607, 0.9506]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5025, 0.8773]\", \"lift_point\": \"[0.5025, 0.8773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.9506]\", \"lift_point\": \"[0.0607, 0.9506]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.081, 0.6107]\", \"lift_point\": \"[0.081, 0.6107]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5025, 0.8773]\", \"lift_point\": \"[0.5025, 0.8773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.9506]\", \"lift_point\": \"[0.0607, 0.9506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.081, 0.6107]\", \"lift_point\": \"[0.081, 0.6107]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated pizza restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5025, 0.8773]\", \"lift_point\": \"[0.5025, 0.8773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.9506]\", \"lift_point\": \"[0.0607, 0.9506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.081, 0.6107]\", \"lift_point\": \"[0.081, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated pizza restaurants\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.7405]\", \"lift_point\": \"[0.1292, 0.7405]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5025, 0.8773]\", \"lift_point\": \"[0.5025, 0.8773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0607, 0.9506]\", \"lift_point\": \"[0.0607, 0.9506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.081, 0.6107]\", \"lift_point\": \"[0.081, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated pizza restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1292, 0.7405]\", \"lift_point\": \"[0.1292, 0.7405]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22481_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.518, 0.6939]\", \"lift_point\": \"[0.518, 0.6939]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.518, 0.6939]\", \"lift_point\": \"[0.518, 0.6939]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2065, 0.761]\", \"lift_point\": \"[0.2065, 0.761]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.518, 0.6939]\", \"lift_point\": \"[0.518, 0.6939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2065, 0.761]\", \"lift_point\": \"[0.2065, 0.761]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9279]\", \"lift_point\": \"[0.058, 0.9279]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.518, 0.6939]\", \"lift_point\": \"[0.518, 0.6939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2065, 0.761]\", \"lift_point\": \"[0.2065, 0.761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9279]\", \"lift_point\": \"[0.058, 0.9279]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.518, 0.6939]\", \"lift_point\": \"[0.518, 0.6939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2065, 0.761]\", \"lift_point\": \"[0.2065, 0.761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9279]\", \"lift_point\": \"[0.058, 0.9279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.6172]\", \"lift_point\": \"[0.1221, 0.6172]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.518, 0.6939]\", \"lift_point\": \"[0.518, 0.6939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2065, 0.761]\", \"lift_point\": \"[0.2065, 0.761]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.9279]\", \"lift_point\": \"[0.058, 0.9279]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.6172]\", \"lift_point\": \"[0.1221, 0.6172]\", \"typed_text\": \"\" Goal: Find the nearest grocery store>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22571_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22585_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22585_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22585_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22585_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2369, 0.5183]\", \"lift_point\": \"[0.2369, 0.5183]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22585_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2369, 0.5183]\", \"lift_point\": \"[0.2369, 0.5183]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22585_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7626, 0.6217]\", \"lift_point\": \"[0.7626, 0.6217]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6247, 0.8487]\", \"lift_point\": \"[0.6247, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.185, 0.6066]\", \"lift_point\": \"[0.185, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.7814]\", \"lift_point\": \"[0.0556, 0.7814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"taylor swift\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.3316]\", \"lift_point\": \"[0.1125, 0.3316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.2994]\", \"lift_point\": \"[0.1715, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0935, 0.3302]\", \"lift_point\": \"[0.0935, 0.3302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.7536]\", \"lift_point\": \"[0.1724, 0.7536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2217, 0.9143]\", \"lift_point\": \"[0.2217, 0.9143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.514, 0.942]\", \"lift_point\": \"[0.514, 0.942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.514, 0.942]\", \"lift_point\": \"[0.514, 0.942]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22676_33.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.333]\", \"lift_point\": \"[0.0721, 0.333]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.333]\", \"lift_point\": \"[0.0721, 0.333]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nyc to london flights\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.333]\", \"lift_point\": \"[0.0721, 0.333]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nyc to london flights\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5506]\", \"lift_point\": \"[0.1352, 0.5506]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.333]\", \"lift_point\": \"[0.0721, 0.333]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nyc to london flights\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5506]\", \"lift_point\": \"[0.1352, 0.5506]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.333]\", \"lift_point\": \"[0.0721, 0.333]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nyc to london flights\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5506]\", \"lift_point\": \"[0.1352, 0.5506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.333]\", \"lift_point\": \"[0.0721, 0.333]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nyc to london flights\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5506]\", \"lift_point\": \"[0.1352, 0.5506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7846, 0.5999]\", \"lift_point\": \"[0.7846, 0.5999]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.6724]\", \"lift_point\": \"[0.7807, 0.6724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.333]\", \"lift_point\": \"[0.0721, 0.333]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nyc to london flights\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.5506]\", \"lift_point\": \"[0.1352, 0.5506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7846, 0.5999]\", \"lift_point\": \"[0.7846, 0.5999]\", \"typed_text\": \"\" Goal: Search for flights from NYC to London>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22813_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6931]\", \"lift_point\": \"[0.7731, 0.6931]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22827_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6931]\", \"lift_point\": \"[0.7731, 0.6931]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.4234]\", \"lift_point\": \"[0.0533, 0.4234]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22827_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6931]\", \"lift_point\": \"[0.7731, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.4234]\", \"lift_point\": \"[0.0533, 0.4234]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipstick on sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22827_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6931]\", \"lift_point\": \"[0.7731, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.4234]\", \"lift_point\": \"[0.0533, 0.4234]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipstick on sephora\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1188, 0.544]\", \"lift_point\": \"[0.1188, 0.544]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22827_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6931]\", \"lift_point\": \"[0.7731, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.4234]\", \"lift_point\": \"[0.0533, 0.4234]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipstick on sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1188, 0.544]\", \"lift_point\": \"[0.1188, 0.544]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/22827_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5689]\", \"lift_point\": \"[0.1339, 0.5689]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5689]\", \"lift_point\": \"[0.1339, 0.5689]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5689]\", \"lift_point\": \"[0.1339, 0.5689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5689]\", \"lift_point\": \"[0.1339, 0.5689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5552, 0.619]\", \"lift_point\": \"[0.5552, 0.619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5689]\", \"lift_point\": \"[0.1339, 0.5689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5399, 0.5883]\", \"lift_point\": \"[0.5399, 0.5883]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.451]\", \"lift_point\": \"[0.8935, 0.451]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5689]\", \"lift_point\": \"[0.1339, 0.5689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5399, 0.5883]\", \"lift_point\": \"[0.5399, 0.5883]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6914, 0.9415]\", \"lift_point\": \"[0.6914, 0.9415]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.5689]\", \"lift_point\": \"[0.1339, 0.5689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5399, 0.5883]\", \"lift_point\": \"[0.5399, 0.5883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6914, 0.9415]\", \"lift_point\": \"[0.6914, 0.9415]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23012_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Atlanta\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Atlanta\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Atlanta\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1183, 0.4854]\", \"lift_point\": \"[0.1183, 0.4854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Atlanta\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1183, 0.4854]\", \"lift_point\": \"[0.1183, 0.4854]\", \"typed_text\": \"\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Atlanta\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1183, 0.4854]\", \"lift_point\": \"[0.1183, 0.4854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Atlanta\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1183, 0.4854]\", \"lift_point\": \"[0.1183, 0.4854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4084, 0.4501]\", \"lift_point\": \"[0.4084, 0.4501]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3662]\", \"lift_point\": \"[0.9011, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Atlanta\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1183, 0.4854]\", \"lift_point\": \"[0.1183, 0.4854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4084, 0.4501]\", \"lift_point\": \"[0.4084, 0.4501]\", \"typed_text\": \"\" Goal: Search for hotels in Atlanta>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23086_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8101, 0.681]\", \"lift_point\": \"[0.8101, 0.681]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8101, 0.681]\", \"lift_point\": \"[0.8101, 0.681]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.6047]\", \"lift_point\": \"[0.8961, 0.6047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8101, 0.681]\", \"lift_point\": \"[0.8101, 0.681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.6047]\", \"lift_point\": \"[0.8961, 0.6047]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.8502]\", \"lift_point\": \"[0.887, 0.8502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8101, 0.681]\", \"lift_point\": \"[0.8101, 0.681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.6047]\", \"lift_point\": \"[0.8961, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.8502]\", \"lift_point\": \"[0.887, 0.8502]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.4917]\", \"lift_point\": \"[0.3197, 0.4917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8101, 0.681]\", \"lift_point\": \"[0.8101, 0.681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.6047]\", \"lift_point\": \"[0.8961, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.8502]\", \"lift_point\": \"[0.887, 0.8502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.4917]\", \"lift_point\": \"[0.3197, 0.4917]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurants in las vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8101, 0.681]\", \"lift_point\": \"[0.8101, 0.681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.6047]\", \"lift_point\": \"[0.8961, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.8502]\", \"lift_point\": \"[0.887, 0.8502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.4917]\", \"lift_point\": \"[0.3197, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurants in las vegas\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5474, 0.6241]\", \"lift_point\": \"[0.5474, 0.6241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8101, 0.681]\", \"lift_point\": \"[0.8101, 0.681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.6047]\", \"lift_point\": \"[0.8961, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.8502]\", \"lift_point\": \"[0.887, 0.8502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.4917]\", \"lift_point\": \"[0.3197, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurants in las vegas\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23104_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23231_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23231_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.1521]\", \"lift_point\": \"[0.5003, 0.1521]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23231_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.1521]\", \"lift_point\": \"[0.5003, 0.1521]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23231_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3579, 0.5075]\", \"lift_point\": \"[0.3579, 0.5075]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3579, 0.5075]\", \"lift_point\": \"[0.3579, 0.5075]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.494]\", \"lift_point\": \"[0.9018, 0.494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3579, 0.5075]\", \"lift_point\": \"[0.3579, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.494]\", \"lift_point\": \"[0.9018, 0.494]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9017]\", \"lift_point\": \"[0.8946, 0.9017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3579, 0.5075]\", \"lift_point\": \"[0.3579, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.494]\", \"lift_point\": \"[0.9018, 0.494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9017]\", \"lift_point\": \"[0.8946, 0.9017]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2867, 0.5688]\", \"lift_point\": \"[0.2867, 0.5688]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3579, 0.5075]\", \"lift_point\": \"[0.3579, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.494]\", \"lift_point\": \"[0.9018, 0.494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9017]\", \"lift_point\": \"[0.8946, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2867, 0.5688]\", \"lift_point\": \"[0.2867, 0.5688]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3579, 0.5075]\", \"lift_point\": \"[0.3579, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.494]\", \"lift_point\": \"[0.9018, 0.494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9017]\", \"lift_point\": \"[0.8946, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2867, 0.5688]\", \"lift_point\": \"[0.2867, 0.5688]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3579, 0.5075]\", \"lift_point\": \"[0.3579, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.494]\", \"lift_point\": \"[0.9018, 0.494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9017]\", \"lift_point\": \"[0.8946, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2867, 0.5688]\", \"lift_point\": \"[0.2867, 0.5688]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good French restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23239_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3545, 0.5048]\", \"lift_point\": \"[0.3545, 0.5048]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3545, 0.5048]\", \"lift_point\": \"[0.3545, 0.5048]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.3913]\", \"lift_point\": \"[0.0537, 0.3913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3545, 0.5048]\", \"lift_point\": \"[0.3545, 0.5048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.3913]\", \"lift_point\": \"[0.0537, 0.3913]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how much does a 2*4*8 board cost at Lowes?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3545, 0.5048]\", \"lift_point\": \"[0.3545, 0.5048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.3913]\", \"lift_point\": \"[0.0537, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how much does a 2*4*8 board cost at Lowes?\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5819, 0.8652]\", \"lift_point\": \"[0.5819, 0.8652]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3545, 0.5048]\", \"lift_point\": \"[0.3545, 0.5048]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.3913]\", \"lift_point\": \"[0.0537, 0.3913]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how much does a 2*4*8 board cost at Lowes?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23264_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.5091]\", \"lift_point\": \"[0.359, 0.5091]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.5091]\", \"lift_point\": \"[0.359, 0.5091]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.5059]\", \"lift_point\": \"[0.3872, 0.5059]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.7098]\", \"lift_point\": \"[0.7868, 0.7098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.5091]\", \"lift_point\": \"[0.359, 0.5091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.5059]\", \"lift_point\": \"[0.3872, 0.5059]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.508]\", \"lift_point\": \"[0.0757, 0.508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.5091]\", \"lift_point\": \"[0.359, 0.5091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.5059]\", \"lift_point\": \"[0.3872, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.508]\", \"lift_point\": \"[0.0757, 0.508]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.941]\", \"lift_point\": \"[0.0721, 0.941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.5091]\", \"lift_point\": \"[0.359, 0.5091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.5059]\", \"lift_point\": \"[0.3872, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.508]\", \"lift_point\": \"[0.0757, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.941]\", \"lift_point\": \"[0.0721, 0.941]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Miami?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.6946]\", \"lift_point\": \"[0.793, 0.6946]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.5091]\", \"lift_point\": \"[0.359, 0.5091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.5059]\", \"lift_point\": \"[0.3872, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.508]\", \"lift_point\": \"[0.0757, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.941]\", \"lift_point\": \"[0.0721, 0.941]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Miami?\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.6816]\", \"lift_point\": \"[0.1463, 0.6816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7187, 0.7899]\", \"lift_point\": \"[0.7187, 0.7899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.359, 0.5091]\", \"lift_point\": \"[0.359, 0.5091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.5059]\", \"lift_point\": \"[0.3872, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.508]\", \"lift_point\": \"[0.0757, 0.508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.941]\", \"lift_point\": \"[0.0721, 0.941]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Miami?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.6816]\", \"lift_point\": \"[0.1463, 0.6816]\", \"typed_text\": \"\" Goal: What's a good restaurant in Miami?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23419_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23446_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3638, 0.4848]\", \"lift_point\": \"[0.3638, 0.4848]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23446_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3638, 0.4848]\", \"lift_point\": \"[0.3638, 0.4848]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3774, 0.5146]\", \"lift_point\": \"[0.3774, 0.5146]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23446_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3638, 0.4848]\", \"lift_point\": \"[0.3638, 0.4848]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3774, 0.5146]\", \"lift_point\": \"[0.3774, 0.5146]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.4674]\", \"lift_point\": \"[0.0623, 0.4674]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23446_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3638, 0.4848]\", \"lift_point\": \"[0.3638, 0.4848]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3774, 0.5146]\", \"lift_point\": \"[0.3774, 0.5146]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.4674]\", \"lift_point\": \"[0.0623, 0.4674]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Boston?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23446_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3638, 0.4848]\", \"lift_point\": \"[0.3638, 0.4848]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3774, 0.5146]\", \"lift_point\": \"[0.3774, 0.5146]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.4674]\", \"lift_point\": \"[0.0623, 0.4674]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Boston?\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1518, 0.5023]\", \"lift_point\": \"[0.1518, 0.5023]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23446_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3638, 0.4848]\", \"lift_point\": \"[0.3638, 0.4848]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3774, 0.5146]\", \"lift_point\": \"[0.3774, 0.5146]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.4674]\", \"lift_point\": \"[0.0623, 0.4674]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Boston?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1518, 0.5023]\", \"lift_point\": \"[0.1518, 0.5023]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23446_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the downloads>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.4897]\", \"lift_point\": \"[0.7521, 0.4897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.4897]\", \"lift_point\": \"[0.7521, 0.4897]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.4897]\", \"lift_point\": \"[0.7521, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.4897]\", \"lift_point\": \"[0.7521, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3459, 0.6862]\", \"lift_point\": \"[0.3459, 0.6862]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.4897]\", \"lift_point\": \"[0.7521, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3459, 0.6862]\", \"lift_point\": \"[0.3459, 0.6862]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.9257]\", \"lift_point\": \"[0.0627, 0.9257]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.459]\", \"lift_point\": \"[0.7521, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7521, 0.4897]\", \"lift_point\": \"[0.7521, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3459, 0.6862]\", \"lift_point\": \"[0.3459, 0.6862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.9257]\", \"lift_point\": \"[0.0627, 0.9257]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23561_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new skincare product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23567_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.701]\", \"lift_point\": \"[0.3442, 0.701]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23567_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.701]\", \"lift_point\": \"[0.3442, 0.701]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.5153]\", \"lift_point\": \"[0.062, 0.5153]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23567_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.701]\", \"lift_point\": \"[0.3442, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.5153]\", \"lift_point\": \"[0.062, 0.5153]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new skincare product\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23567_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.701]\", \"lift_point\": \"[0.3442, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.5153]\", \"lift_point\": \"[0.062, 0.5153]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new skincare product\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1529, 0.6008]\", \"lift_point\": \"[0.1529, 0.6008]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23567_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.701]\", \"lift_point\": \"[0.3442, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.5153]\", \"lift_point\": \"[0.062, 0.5153]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new skincare product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1529, 0.6008]\", \"lift_point\": \"[0.1529, 0.6008]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23567_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4426]\", \"lift_point\": \"[0.0646, 0.4426]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4426]\", \"lift_point\": \"[0.0646, 0.4426]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9082]\", \"lift_point\": \"[0.056, 0.9082]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4426]\", \"lift_point\": \"[0.0646, 0.4426]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9082]\", \"lift_point\": \"[0.056, 0.9082]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.8296]\", \"lift_point\": \"[0.6098, 0.8296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4426]\", \"lift_point\": \"[0.0646, 0.4426]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9082]\", \"lift_point\": \"[0.056, 0.9082]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil?\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.5095]\", \"lift_point\": \"[0.1411, 0.5095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4426]\", \"lift_point\": \"[0.0646, 0.4426]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9082]\", \"lift_point\": \"[0.056, 0.9082]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.5095]\", \"lift_point\": \"[0.1411, 0.5095]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1553, 0.4982]\", \"lift_point\": \"[0.1553, 0.4982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.334, 0.8355]\", \"lift_point\": \"[0.334, 0.8355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4426]\", \"lift_point\": \"[0.0646, 0.4426]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9082]\", \"lift_point\": \"[0.056, 0.9082]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.5095]\", \"lift_point\": \"[0.1411, 0.5095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1553, 0.4982]\", \"lift_point\": \"[0.1553, 0.4982]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1415, 0.3887]\", \"lift_point\": \"[0.1415, 0.3887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.6863]\", \"lift_point\": \"[0.336, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4426]\", \"lift_point\": \"[0.0646, 0.4426]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.056, 0.9082]\", \"lift_point\": \"[0.056, 0.9082]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.5095]\", \"lift_point\": \"[0.1411, 0.5095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1553, 0.4982]\", \"lift_point\": \"[0.1553, 0.4982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1415, 0.3887]\", \"lift_point\": \"[0.1415, 0.3887]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23603_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.8981]\", \"lift_point\": \"[0.0587, 0.8981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.8981]\", \"lift_point\": \"[0.0587, 0.8981]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.8981]\", \"lift_point\": \"[0.0587, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Chicago\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1296, 0.3482]\", \"lift_point\": \"[0.1296, 0.3482]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.8981]\", \"lift_point\": \"[0.0587, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1296, 0.3482]\", \"lift_point\": \"[0.1296, 0.3482]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3223, 0.42]\", \"lift_point\": \"[0.3223, 0.42]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.8981]\", \"lift_point\": \"[0.0587, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1296, 0.3482]\", \"lift_point\": \"[0.1296, 0.3482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3223, 0.42]\", \"lift_point\": \"[0.3223, 0.42]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8873, 0.5619]\", \"lift_point\": \"[0.8873, 0.5619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.8981]\", \"lift_point\": \"[0.0587, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1296, 0.3482]\", \"lift_point\": \"[0.1296, 0.3482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3223, 0.42]\", \"lift_point\": \"[0.3223, 0.42]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8873, 0.5619]\", \"lift_point\": \"[0.8873, 0.5619]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9061, 0.8273]\", \"lift_point\": \"[0.9061, 0.8273]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.1508]\", \"lift_point\": \"[0.6314, 0.1508]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.8981]\", \"lift_point\": \"[0.0587, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1296, 0.3482]\", \"lift_point\": \"[0.1296, 0.3482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3223, 0.42]\", \"lift_point\": \"[0.3223, 0.42]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8873, 0.5619]\", \"lift_point\": \"[0.8873, 0.5619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9061, 0.8273]\", \"lift_point\": \"[0.9061, 0.8273]\", \"typed_text\": \"\" Goal: Search for hotels in Chicago>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23713_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.1743]\", \"lift_point\": \"[0.6212, 0.1743]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.1743]\", \"lift_point\": \"[0.6212, 0.1743]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.6615]\", \"lift_point\": \"[0.1286, 0.6615]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.1743]\", \"lift_point\": \"[0.6212, 0.1743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.6615]\", \"lift_point\": \"[0.1286, 0.6615]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.912]\", \"lift_point\": \"[0.067, 0.912]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.1743]\", \"lift_point\": \"[0.6212, 0.1743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.6615]\", \"lift_point\": \"[0.1286, 0.6615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.912]\", \"lift_point\": \"[0.067, 0.912]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.1743]\", \"lift_point\": \"[0.6212, 0.1743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.6615]\", \"lift_point\": \"[0.1286, 0.6615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.912]\", \"lift_point\": \"[0.067, 0.912]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.6514]\", \"lift_point\": \"[0.116, 0.6514]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.1743]\", \"lift_point\": \"[0.6212, 0.1743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.6615]\", \"lift_point\": \"[0.1286, 0.6615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.912]\", \"lift_point\": \"[0.067, 0.912]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.6514]\", \"lift_point\": \"[0.116, 0.6514]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.3714]\", \"lift_point\": \"[0.6221, 0.3714]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.1743]\", \"lift_point\": \"[0.6212, 0.1743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.6615]\", \"lift_point\": \"[0.1286, 0.6615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.912]\", \"lift_point\": \"[0.067, 0.912]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Zurich to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.6514]\", \"lift_point\": \"[0.116, 0.6514]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.3714]\", \"lift_point\": \"[0.6221, 0.3714]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23754_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the downloads>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23833_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.5329]\", \"lift_point\": \"[0.8945, 0.5329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23833_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.5329]\", \"lift_point\": \"[0.8945, 0.5329]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the downloads\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23833_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.5329]\", \"lift_point\": \"[0.8945, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the downloads\" Goal: Open the downloads>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23833_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.5329]\", \"lift_point\": \"[0.8945, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Open the downloads\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23833_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3893, 0.7047]\", \"lift_point\": \"[0.3893, 0.7047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3893, 0.7047]\", \"lift_point\": \"[0.3893, 0.7047]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.516]\", \"lift_point\": \"[0.8764, 0.516]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3893, 0.7047]\", \"lift_point\": \"[0.3893, 0.7047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.516]\", \"lift_point\": \"[0.8764, 0.516]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6223, 0.3817]\", \"lift_point\": \"[0.6223, 0.3817]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3893, 0.7047]\", \"lift_point\": \"[0.3893, 0.7047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.516]\", \"lift_point\": \"[0.8764, 0.516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6223, 0.3817]\", \"lift_point\": \"[0.6223, 0.3817]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.5021]\", \"lift_point\": \"[0.4137, 0.5021]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3893, 0.7047]\", \"lift_point\": \"[0.3893, 0.7047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.516]\", \"lift_point\": \"[0.8764, 0.516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6223, 0.3817]\", \"lift_point\": \"[0.6223, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.5021]\", \"lift_point\": \"[0.4137, 0.5021]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7037, 0.7908]\", \"lift_point\": \"[0.7037, 0.7908]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3893, 0.7047]\", \"lift_point\": \"[0.3893, 0.7047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.516]\", \"lift_point\": \"[0.8764, 0.516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6223, 0.3817]\", \"lift_point\": \"[0.6223, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.5021]\", \"lift_point\": \"[0.4137, 0.5021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7037, 0.7908]\", \"lift_point\": \"[0.7037, 0.7908]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/23954_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7451, 0.302]\", \"lift_point\": \"[0.7451, 0.302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2603, 0.7206]\", \"lift_point\": \"[0.2603, 0.7206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.363]\", \"lift_point\": \"[0.0666, 0.363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1639, 0.921]\", \"lift_point\": \"[0.1639, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.363]\", \"lift_point\": \"[0.0666, 0.363]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"game trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.363]\", \"lift_point\": \"[0.0666, 0.363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3115]\", \"lift_point\": \"[0.3207, 0.3115]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.363]\", \"lift_point\": \"[0.0666, 0.363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3115]\", \"lift_point\": \"[0.3207, 0.3115]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2571, 0.7836]\", \"lift_point\": \"[0.2571, 0.7836]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.435, 0.3619]\", \"lift_point\": \"[0.435, 0.3619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0666, 0.363]\", \"lift_point\": \"[0.0666, 0.363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3115]\", \"lift_point\": \"[0.3207, 0.3115]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2571, 0.7836]\", \"lift_point\": \"[0.2571, 0.7836]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24054_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calendar app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4052, 0.149]\", \"lift_point\": \"[0.4052, 0.149]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4052, 0.149]\", \"lift_point\": \"[0.4052, 0.149]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4052, 0.149]\", \"lift_point\": \"[0.4052, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4052, 0.149]\", \"lift_point\": \"[0.4052, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4052, 0.149]\", \"lift_point\": \"[0.4052, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8538, 0.524]\", \"lift_point\": \"[0.8538, 0.524]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4052, 0.149]\", \"lift_point\": \"[0.4052, 0.149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8825, 0.8932]\", \"lift_point\": \"[0.8825, 0.8932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8538, 0.524]\", \"lift_point\": \"[0.8538, 0.524]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24354_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0752, 0.5699]\", \"lift_point\": \"[0.0752, 0.5699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8209, 0.8498]\", \"lift_point\": \"[0.8209, 0.8498]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8209, 0.8498]\", \"lift_point\": \"[0.8209, 0.8498]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2783, 0.4273]\", \"lift_point\": \"[0.2783, 0.4273]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8209, 0.8498]\", \"lift_point\": \"[0.8209, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1515, 0.9091]\", \"lift_point\": \"[0.1515, 0.9091]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4492, 0.1148]\", \"lift_point\": \"[0.4492, 0.1148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8209, 0.8498]\", \"lift_point\": \"[0.8209, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1515, 0.9091]\", \"lift_point\": \"[0.1515, 0.9091]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6437, 0.4219]\", \"lift_point\": \"[0.6437, 0.4219]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6379]\", \"lift_point\": \"[0.4832, 0.6379]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8209, 0.8498]\", \"lift_point\": \"[0.8209, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1515, 0.9091]\", \"lift_point\": \"[0.1515, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6437, 0.4219]\", \"lift_point\": \"[0.6437, 0.4219]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.8037]\", \"lift_point\": \"[0.7387, 0.8037]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8209, 0.8498]\", \"lift_point\": \"[0.8209, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1515, 0.9091]\", \"lift_point\": \"[0.1515, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6437, 0.4219]\", \"lift_point\": \"[0.6437, 0.4219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8944]\", \"lift_point\": \"[0.1595, 0.8944]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4438, 0.3849]\", \"lift_point\": \"[0.4438, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8209, 0.8498]\", \"lift_point\": \"[0.8209, 0.8498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1515, 0.9091]\", \"lift_point\": \"[0.1515, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6437, 0.4219]\", \"lift_point\": \"[0.6437, 0.4219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8944]\", \"lift_point\": \"[0.1595, 0.8944]\", \"typed_text\": \"\" Goal: What's on Reddit>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24436_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.757, 0.3336]\", \"lift_point\": \"[0.757, 0.3336]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24520_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.757, 0.3336]\", \"lift_point\": \"[0.757, 0.3336]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24520_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.757, 0.3336]\", \"lift_point\": \"[0.757, 0.3336]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7443, 0.2863]\", \"lift_point\": \"[0.7443, 0.2863]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24520_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.757, 0.3336]\", \"lift_point\": \"[0.757, 0.3336]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7443, 0.2863]\", \"lift_point\": \"[0.7443, 0.2863]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.9381]\", \"lift_point\": \"[0.0729, 0.9381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24520_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.757, 0.3336]\", \"lift_point\": \"[0.757, 0.3336]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7443, 0.2863]\", \"lift_point\": \"[0.7443, 0.2863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.9381]\", \"lift_point\": \"[0.0729, 0.9381]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1679, 0.5016]\", \"lift_point\": \"[0.1679, 0.5016]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24520_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.757, 0.3336]\", \"lift_point\": \"[0.757, 0.3336]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7443, 0.2863]\", \"lift_point\": \"[0.7443, 0.2863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.9381]\", \"lift_point\": \"[0.0729, 0.9381]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1679, 0.5016]\", \"lift_point\": \"[0.1679, 0.5016]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24520_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.8244]\", \"lift_point\": \"[0.2144, 0.8244]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.8244]\", \"lift_point\": \"[0.2144, 0.8244]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7568, 0.679]\", \"lift_point\": \"[0.7568, 0.679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.8244]\", \"lift_point\": \"[0.2144, 0.8244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.7035]\", \"lift_point\": \"[0.7722, 0.7035]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.8244]\", \"lift_point\": \"[0.2144, 0.8244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0808, 0.7656]\", \"lift_point\": \"[0.0808, 0.7656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.8244]\", \"lift_point\": \"[0.2144, 0.8244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9075]\", \"lift_point\": \"[0.0762, 0.9075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.8244]\", \"lift_point\": \"[0.2144, 0.8244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1807, 0.8051]\", \"lift_point\": \"[0.1807, 0.8051]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2144, 0.8244]\", \"lift_point\": \"[0.2144, 0.8244]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1807, 0.8051]\", \"lift_point\": \"[0.1807, 0.8051]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24554_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24572_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24572_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9036, 0.4394]\", \"lift_point\": \"[0.9036, 0.4394]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24593_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9036, 0.4394]\", \"lift_point\": \"[0.9036, 0.4394]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8892, 0.508]\", \"lift_point\": \"[0.8892, 0.508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24593_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9036, 0.4394]\", \"lift_point\": \"[0.9036, 0.4394]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8892, 0.508]\", \"lift_point\": \"[0.8892, 0.508]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new nail polish\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24593_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9036, 0.4394]\", \"lift_point\": \"[0.9036, 0.4394]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8892, 0.508]\", \"lift_point\": \"[0.8892, 0.508]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new nail polish\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1798, 0.3692]\", \"lift_point\": \"[0.1798, 0.3692]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24593_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9036, 0.4394]\", \"lift_point\": \"[0.9036, 0.4394]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8892, 0.508]\", \"lift_point\": \"[0.8892, 0.508]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new nail polish\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1798, 0.3692]\", \"lift_point\": \"[0.1798, 0.3692]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24593_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Hisense TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24611_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\" Goal: What's the price of the Hisense TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Hisense TV?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24611_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Hisense TV?\" Goal: What's the price of the Hisense TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.443]\", \"lift_point\": \"[0.117, 0.443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24611_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Hisense TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.443]\", \"lift_point\": \"[0.117, 0.443]\", \"typed_text\": \"\" Goal: What's the price of the Hisense TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.1982]\", \"lift_point\": \"[0.3452, 0.1982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24611_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Hisense TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.443]\", \"lift_point\": \"[0.117, 0.443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.1982]\", \"lift_point\": \"[0.3452, 0.1982]\", \"typed_text\": \"\" Goal: What's the price of the Hisense TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7584, 0.939]\", \"lift_point\": \"[0.7584, 0.939]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24611_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Hisense TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.443]\", \"lift_point\": \"[0.117, 0.443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.1982]\", \"lift_point\": \"[0.3452, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7584, 0.939]\", \"lift_point\": \"[0.7584, 0.939]\", \"typed_text\": \"\" Goal: What's the price of the Hisense TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.9327]\", \"lift_point\": \"[0.749, 0.9327]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24611_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.884, 0.53]\", \"lift_point\": \"[0.884, 0.53]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Hisense TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.443]\", \"lift_point\": \"[0.117, 0.443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.1982]\", \"lift_point\": \"[0.3452, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7584, 0.939]\", \"lift_point\": \"[0.7584, 0.939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.749, 0.9327]\", \"lift_point\": \"[0.749, 0.9327]\", \"typed_text\": \"\" Goal: What's the price of the Hisense TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24611_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8561, 0.9226]\", \"lift_point\": \"[0.8561, 0.9226]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8561, 0.9226]\", \"lift_point\": \"[0.8561, 0.9226]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.0684]\", \"lift_point\": \"[0.0608, 0.0684]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8561, 0.9226]\", \"lift_point\": \"[0.8561, 0.9226]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.0684]\", \"lift_point\": \"[0.0608, 0.0684]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0487, 0.8138]\", \"lift_point\": \"[0.0487, 0.8138]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8561, 0.9226]\", \"lift_point\": \"[0.8561, 0.9226]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.0684]\", \"lift_point\": \"[0.0608, 0.0684]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0487, 0.8138]\", \"lift_point\": \"[0.0487, 0.8138]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Bruno Mars video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8561, 0.9226]\", \"lift_point\": \"[0.8561, 0.9226]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.0684]\", \"lift_point\": \"[0.0608, 0.0684]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0487, 0.8138]\", \"lift_point\": \"[0.0487, 0.8138]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Bruno Mars video\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.11, 0.416]\", \"lift_point\": \"[0.11, 0.416]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8561, 0.9226]\", \"lift_point\": \"[0.8561, 0.9226]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.0684]\", \"lift_point\": \"[0.0608, 0.0684]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0487, 0.8138]\", \"lift_point\": \"[0.0487, 0.8138]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Bruno Mars video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.11, 0.416]\", \"lift_point\": \"[0.11, 0.416]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3355, 0.48]\", \"lift_point\": \"[0.3355, 0.48]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7737, 0.8688]\", \"lift_point\": \"[0.7737, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.0735]\", \"lift_point\": \"[0.0472, 0.0735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8561, 0.9226]\", \"lift_point\": \"[0.8561, 0.9226]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.0684]\", \"lift_point\": \"[0.0608, 0.0684]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0487, 0.8138]\", \"lift_point\": \"[0.0487, 0.8138]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Bruno Mars video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.11, 0.416]\", \"lift_point\": \"[0.11, 0.416]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3355, 0.48]\", \"lift_point\": \"[0.3355, 0.48]\", \"typed_text\": \"\" Goal: Play the new Bruno Mars video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24635_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5115, 0.5001]\", \"lift_point\": \"[0.5115, 0.5001]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5115, 0.5001]\", \"lift_point\": \"[0.5115, 0.5001]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5463, 0.4976]\", \"lift_point\": \"[0.5463, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5115, 0.5001]\", \"lift_point\": \"[0.5115, 0.5001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.2441]\", \"lift_point\": \"[0.8229, 0.2441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5115, 0.5001]\", \"lift_point\": \"[0.5115, 0.5001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.9121]\", \"lift_point\": \"[0.6879, 0.9121]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1028, 0.4618]\", \"lift_point\": \"[0.1028, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5115, 0.5001]\", \"lift_point\": \"[0.5115, 0.5001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.9121]\", \"lift_point\": \"[0.6879, 0.9121]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.678, 0.8882]\", \"lift_point\": \"[0.678, 0.8882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5115, 0.5001]\", \"lift_point\": \"[0.5115, 0.5001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6879, 0.9121]\", \"lift_point\": \"[0.6879, 0.9121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.678, 0.8882]\", \"lift_point\": \"[0.678, 0.8882]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24843_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24934_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5487, 0.4375]\", \"lift_point\": \"[0.5487, 0.4375]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24934_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5487, 0.4375]\", \"lift_point\": \"[0.5487, 0.4375]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5416, 0.5029]\", \"lift_point\": \"[0.5416, 0.5029]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24934_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5487, 0.4375]\", \"lift_point\": \"[0.5487, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5416, 0.5029]\", \"lift_point\": \"[0.5416, 0.5029]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.4914]\", \"lift_point\": \"[0.1082, 0.4914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24934_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5487, 0.4375]\", \"lift_point\": \"[0.5487, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5416, 0.5029]\", \"lift_point\": \"[0.5416, 0.5029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.4914]\", \"lift_point\": \"[0.1082, 0.4914]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24934_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5487, 0.4375]\", \"lift_point\": \"[0.5487, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5416, 0.5029]\", \"lift_point\": \"[0.5416, 0.5029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.4914]\", \"lift_point\": \"[0.1082, 0.4914]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24934_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5487, 0.4375]\", \"lift_point\": \"[0.5487, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5416, 0.5029]\", \"lift_point\": \"[0.5416, 0.5029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.4914]\", \"lift_point\": \"[0.1082, 0.4914]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24934_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4956, 0.4605]\", \"lift_point\": \"[0.4956, 0.4605]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4956, 0.4605]\", \"lift_point\": \"[0.4956, 0.4605]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1153, 0.6048]\", \"lift_point\": \"[0.1153, 0.6048]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4956, 0.4605]\", \"lift_point\": \"[0.4956, 0.4605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1153, 0.6048]\", \"lift_point\": \"[0.1153, 0.6048]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Best Buy?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4956, 0.4605]\", \"lift_point\": \"[0.4956, 0.4605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1153, 0.6048]\", \"lift_point\": \"[0.1153, 0.6048]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Best Buy?\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3003, 0.2088]\", \"lift_point\": \"[0.3003, 0.2088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4956, 0.4605]\", \"lift_point\": \"[0.4956, 0.4605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1153, 0.6048]\", \"lift_point\": \"[0.1153, 0.6048]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Best Buy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Best Buy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/24995_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.4928]\", \"lift_point\": \"[0.5003, 0.4928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.4928]\", \"lift_point\": \"[0.5003, 0.4928]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.5292]\", \"lift_point\": \"[0.1093, 0.5292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.4928]\", \"lift_point\": \"[0.5003, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.5292]\", \"lift_point\": \"[0.1093, 0.5292]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1016, 0.8894]\", \"lift_point\": \"[0.1016, 0.8894]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.4928]\", \"lift_point\": \"[0.5003, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.5292]\", \"lift_point\": \"[0.1093, 0.5292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1016, 0.8894]\", \"lift_point\": \"[0.1016, 0.8894]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Seoul?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.4928]\", \"lift_point\": \"[0.5003, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.5292]\", \"lift_point\": \"[0.1093, 0.5292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1016, 0.8894]\", \"lift_point\": \"[0.1016, 0.8894]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Seoul?\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5003, 0.4928]\", \"lift_point\": \"[0.5003, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.5292]\", \"lift_point\": \"[0.1093, 0.5292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1016, 0.8894]\", \"lift_point\": \"[0.1016, 0.8894]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Seoul?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25004_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this month?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3687, 0.6377]\", \"lift_point\": \"[0.3687, 0.6377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3687, 0.6377]\", \"lift_point\": \"[0.3687, 0.6377]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3687, 0.6377]\", \"lift_point\": \"[0.3687, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3687, 0.6377]\", \"lift_point\": \"[0.3687, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3687, 0.6377]\", \"lift_point\": \"[0.3687, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2971, 0.4607]\", \"lift_point\": \"[0.2971, 0.4607]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3687, 0.6377]\", \"lift_point\": \"[0.3687, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2971, 0.4607]\", \"lift_point\": \"[0.2971, 0.4607]\", \"typed_text\": \"\" Goal: What's the news this month?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25029_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3467, 0.6157]\", \"lift_point\": \"[0.3467, 0.6157]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3467, 0.6157]\", \"lift_point\": \"[0.3467, 0.6157]\", \"typed_text\": \"\" Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1943, 0.6497]\", \"lift_point\": \"[0.1943, 0.6497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3467, 0.6157]\", \"lift_point\": \"[0.3467, 0.6157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1943, 0.6497]\", \"lift_point\": \"[0.1943, 0.6497]\", \"typed_text\": \"\" Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.6579]\", \"lift_point\": \"[0.0498, 0.6579]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3467, 0.6157]\", \"lift_point\": \"[0.3467, 0.6157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1943, 0.6497]\", \"lift_point\": \"[0.1943, 0.6497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.6579]\", \"lift_point\": \"[0.0498, 0.6579]\", \"typed_text\": \"\" Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Jamaica?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3467, 0.6157]\", \"lift_point\": \"[0.3467, 0.6157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1943, 0.6497]\", \"lift_point\": \"[0.1943, 0.6497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.6579]\", \"lift_point\": \"[0.0498, 0.6579]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Jamaica?\" Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3467, 0.6157]\", \"lift_point\": \"[0.3467, 0.6157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1943, 0.6497]\", \"lift_point\": \"[0.1943, 0.6497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.6579]\", \"lift_point\": \"[0.0498, 0.6579]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Jamaica?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Jamaica?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25172_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25236_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.6233]\", \"lift_point\": \"[0.3566, 0.6233]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25236_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.6233]\", \"lift_point\": \"[0.3566, 0.6233]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.5478]\", \"lift_point\": \"[0.0637, 0.5478]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25236_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.6233]\", \"lift_point\": \"[0.3566, 0.6233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.5478]\", \"lift_point\": \"[0.0637, 0.5478]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in San Francisco?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25236_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.6233]\", \"lift_point\": \"[0.3566, 0.6233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.5478]\", \"lift_point\": \"[0.0637, 0.5478]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in San Francisco?\" Goal: How much does a 3 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25236_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.6233]\", \"lift_point\": \"[0.3566, 0.6233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.5478]\", \"lift_point\": \"[0.0637, 0.5478]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in San Francisco?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3394, 0.3787]\", \"lift_point\": \"[0.3394, 0.3787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25236_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.6233]\", \"lift_point\": \"[0.3566, 0.6233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.5478]\", \"lift_point\": \"[0.0637, 0.5478]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in San Francisco?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3394, 0.3787]\", \"lift_point\": \"[0.3394, 0.3787]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25236_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3491, 0.6377]\", \"lift_point\": \"[0.3491, 0.6377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3447, 0.5798]\", \"lift_point\": \"[0.3447, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3491, 0.6377]\", \"lift_point\": \"[0.3491, 0.6377]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.4824]\", \"lift_point\": \"[0.0549, 0.4824]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.3801]\", \"lift_point\": \"[0.058, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3491, 0.6377]\", \"lift_point\": \"[0.3491, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.4824]\", \"lift_point\": \"[0.0549, 0.4824]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3491, 0.6377]\", \"lift_point\": \"[0.3491, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.4824]\", \"lift_point\": \"[0.0549, 0.4824]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3491, 0.6377]\", \"lift_point\": \"[0.3491, 0.6377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.4824]\", \"lift_point\": \"[0.0549, 0.4824]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25257_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach cooler?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25304_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach cooler?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3268, 0.5633]\", \"lift_point\": \"[0.3268, 0.5633]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25304_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3268, 0.5633]\", \"lift_point\": \"[0.3268, 0.5633]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach cooler?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0626, 0.5134]\", \"lift_point\": \"[0.0626, 0.5134]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25304_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3268, 0.5633]\", \"lift_point\": \"[0.3268, 0.5633]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0626, 0.5134]\", \"lift_point\": \"[0.0626, 0.5134]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach cooler?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach cooler?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25304_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3268, 0.5633]\", \"lift_point\": \"[0.3268, 0.5633]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0626, 0.5134]\", \"lift_point\": \"[0.0626, 0.5134]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach cooler?\" Goal: Where can I buy a nice beach cooler?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25304_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3268, 0.5633]\", \"lift_point\": \"[0.3268, 0.5633]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0626, 0.5134]\", \"lift_point\": \"[0.0626, 0.5134]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach cooler?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach cooler?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25304_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25384_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3543, 0.8344]\", \"lift_point\": \"[0.3543, 0.8344]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25384_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3543, 0.8344]\", \"lift_point\": \"[0.3543, 0.8344]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25384_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2025, 0.3128]\", \"lift_point\": \"[0.2025, 0.3128]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2025, 0.3128]\", \"lift_point\": \"[0.2025, 0.3128]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0917, 0.1112]\", \"lift_point\": \"[0.0917, 0.1112]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2025, 0.3128]\", \"lift_point\": \"[0.2025, 0.3128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0917, 0.1112]\", \"lift_point\": \"[0.0917, 0.1112]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5295]\", \"lift_point\": \"[0.8789, 0.5295]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2025, 0.3128]\", \"lift_point\": \"[0.2025, 0.3128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0917, 0.1112]\", \"lift_point\": \"[0.0917, 0.1112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5295]\", \"lift_point\": \"[0.8789, 0.5295]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5827, 0.6927]\", \"lift_point\": \"[0.5827, 0.6927]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2025, 0.3128]\", \"lift_point\": \"[0.2025, 0.3128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0917, 0.1112]\", \"lift_point\": \"[0.0917, 0.1112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5295]\", \"lift_point\": \"[0.8789, 0.5295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5827, 0.6927]\", \"lift_point\": \"[0.5827, 0.6927]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4045, 0.5047]\", \"lift_point\": \"[0.4045, 0.5047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2025, 0.3128]\", \"lift_point\": \"[0.2025, 0.3128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0917, 0.1112]\", \"lift_point\": \"[0.0917, 0.1112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5295]\", \"lift_point\": \"[0.8789, 0.5295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5827, 0.6927]\", \"lift_point\": \"[0.5827, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4045, 0.5047]\", \"lift_point\": \"[0.4045, 0.5047]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.7586]\", \"lift_point\": \"[0.6971, 0.7586]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2025, 0.3128]\", \"lift_point\": \"[0.2025, 0.3128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0917, 0.1112]\", \"lift_point\": \"[0.0917, 0.1112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5295]\", \"lift_point\": \"[0.8789, 0.5295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5827, 0.6927]\", \"lift_point\": \"[0.5827, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4045, 0.5047]\", \"lift_point\": \"[0.4045, 0.5047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.7586]\", \"lift_point\": \"[0.6971, 0.7586]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25424_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new incognito window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25490_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new incognito window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3617, 0.4902]\", \"lift_point\": \"[0.3617, 0.4902]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25490_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3617, 0.4902]\", \"lift_point\": \"[0.3617, 0.4902]\", \"typed_text\": \"\" Goal: Open a new incognito window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4429]\", \"lift_point\": \"[0.9117, 0.4429]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25490_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3617, 0.4902]\", \"lift_point\": \"[0.3617, 0.4902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4429]\", \"lift_point\": \"[0.9117, 0.4429]\", \"typed_text\": \"\" Goal: Open a new incognito window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9159, 0.8003]\", \"lift_point\": \"[0.9159, 0.8003]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25490_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3617, 0.4902]\", \"lift_point\": \"[0.3617, 0.4902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4429]\", \"lift_point\": \"[0.9117, 0.4429]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9159, 0.8003]\", \"lift_point\": \"[0.9159, 0.8003]\", \"typed_text\": \"\" Goal: Open a new incognito window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9363]\", \"lift_point\": \"[0.0588, 0.9363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25490_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3617, 0.4902]\", \"lift_point\": \"[0.3617, 0.4902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4429]\", \"lift_point\": \"[0.9117, 0.4429]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9159, 0.8003]\", \"lift_point\": \"[0.9159, 0.8003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9363]\", \"lift_point\": \"[0.0588, 0.9363]\", \"typed_text\": \"\" Goal: Open a new incognito window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1719, 0.7679]\", \"lift_point\": \"[0.1719, 0.7679]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25490_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3617, 0.4902]\", \"lift_point\": \"[0.3617, 0.4902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4429]\", \"lift_point\": \"[0.9117, 0.4429]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9159, 0.8003]\", \"lift_point\": \"[0.9159, 0.8003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.9363]\", \"lift_point\": \"[0.0588, 0.9363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1719, 0.7679]\", \"lift_point\": \"[0.1719, 0.7679]\", \"typed_text\": \"\" Goal: Open a new incognito window in the chrome app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25490_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25557_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8891, 0.1918]\", \"lift_point\": \"[0.8891, 0.1918]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25557_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8891, 0.1918]\", \"lift_point\": \"[0.8891, 0.1918]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Italy\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25557_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8891, 0.1918]\", \"lift_point\": \"[0.8891, 0.1918]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Italy\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25557_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8891, 0.1918]\", \"lift_point\": \"[0.8891, 0.1918]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Italy\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25557_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.8532]\", \"lift_point\": \"[0.6207, 0.8532]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.8532]\", \"lift_point\": \"[0.6207, 0.8532]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.8532]\", \"lift_point\": \"[0.6207, 0.8532]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.7738]\", \"lift_point\": \"[0.0562, 0.7738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.8532]\", \"lift_point\": \"[0.6207, 0.8532]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.7738]\", \"lift_point\": \"[0.0562, 0.7738]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0678, 0.9275]\", \"lift_point\": \"[0.0678, 0.9275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.8532]\", \"lift_point\": \"[0.6207, 0.8532]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.7738]\", \"lift_point\": \"[0.0562, 0.7738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0678, 0.9275]\", \"lift_point\": \"[0.0678, 0.9275]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Amazon Music app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.8532]\", \"lift_point\": \"[0.6207, 0.8532]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.7738]\", \"lift_point\": \"[0.0562, 0.7738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0678, 0.9275]\", \"lift_point\": \"[0.0678, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Amazon Music app\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.8532]\", \"lift_point\": \"[0.6207, 0.8532]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.7738]\", \"lift_point\": \"[0.0562, 0.7738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0678, 0.9275]\", \"lift_point\": \"[0.0678, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Amazon Music app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25593_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25611_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8853, 0.6011]\", \"lift_point\": \"[0.8853, 0.6011]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25611_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8853, 0.6011]\", \"lift_point\": \"[0.8853, 0.6011]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25611_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8853, 0.6011]\", \"lift_point\": \"[0.8853, 0.6011]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25611_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8853, 0.6011]\", \"lift_point\": \"[0.8853, 0.6011]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25611_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8853, 0.6011]\", \"lift_point\": \"[0.8853, 0.6011]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What does the iPhone 8 look like\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25611_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a blue whale?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25679_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4733]\", \"lift_point\": \"[0.8803, 0.4733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25679_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4733]\", \"lift_point\": \"[0.8803, 0.4733]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25679_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4733]\", \"lift_point\": \"[0.8803, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25679_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4733]\", \"lift_point\": \"[0.8803, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25679_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25758_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8345]\", \"lift_point\": \"[0.2994, 0.8345]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25758_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8345]\", \"lift_point\": \"[0.2994, 0.8345]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8373]\", \"lift_point\": \"[0.2994, 0.8373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25758_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8345]\", \"lift_point\": \"[0.2994, 0.8345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8373]\", \"lift_point\": \"[0.2994, 0.8373]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8426, 0.5058]\", \"lift_point\": \"[0.8426, 0.5058]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25758_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8345]\", \"lift_point\": \"[0.2994, 0.8345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8373]\", \"lift_point\": \"[0.2994, 0.8373]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8426, 0.5058]\", \"lift_point\": \"[0.8426, 0.5058]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5408, 0.3006]\", \"lift_point\": \"[0.5408, 0.3006]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25758_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8345]\", \"lift_point\": \"[0.2994, 0.8345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8373]\", \"lift_point\": \"[0.2994, 0.8373]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8426, 0.5058]\", \"lift_point\": \"[0.8426, 0.5058]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5408, 0.3006]\", \"lift_point\": \"[0.5408, 0.3006]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7118, 0.7632]\", \"lift_point\": \"[0.7118, 0.7632]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25758_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8345]\", \"lift_point\": \"[0.2994, 0.8345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2994, 0.8373]\", \"lift_point\": \"[0.2994, 0.8373]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8426, 0.5058]\", \"lift_point\": \"[0.8426, 0.5058]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5408, 0.3006]\", \"lift_point\": \"[0.5408, 0.3006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7118, 0.7632]\", \"lift_point\": \"[0.7118, 0.7632]\", \"typed_text\": \"\" Goal: Set an alarm for 9am>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25758_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8477, 0.6484]\", \"lift_point\": \"[0.8477, 0.6484]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0537, 0.5218]\", \"lift_point\": \"[0.0537, 0.5218]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2129, 0.1286]\", \"lift_point\": \"[0.2129, 0.1286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.6326]\", \"lift_point\": \"[0.0549, 0.6326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Chipotle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.2639]\", \"lift_point\": \"[0.1115, 0.2639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2129, 0.1286]\", \"lift_point\": \"[0.2129, 0.1286]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25835_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1014, 0.94]\", \"lift_point\": \"[0.1014, 0.94]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1014, 0.94]\", \"lift_point\": \"[0.1014, 0.94]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1014, 0.94]\", \"lift_point\": \"[0.1014, 0.94]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8421, 0.6318]\", \"lift_point\": \"[0.8421, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1014, 0.94]\", \"lift_point\": \"[0.1014, 0.94]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0465, 0.4518]\", \"lift_point\": \"[0.0465, 0.4518]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1014, 0.94]\", \"lift_point\": \"[0.1014, 0.94]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the New York Times\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1014, 0.94]\", \"lift_point\": \"[0.1014, 0.94]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8492, 0.2312]\", \"lift_point\": \"[0.8492, 0.2312]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1493, 0.6144]\", \"lift_point\": \"[0.1493, 0.6144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1014, 0.94]\", \"lift_point\": \"[0.1014, 0.94]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8492, 0.2312]\", \"lift_point\": \"[0.8492, 0.2312]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25873_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0435, 0.8443]\", \"lift_point\": \"[0.0435, 0.8443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0435, 0.8443]\", \"lift_point\": \"[0.0435, 0.8443]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Selena Gomez video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0435, 0.8443]\", \"lift_point\": \"[0.0435, 0.8443]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Selena Gomez video\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0435, 0.8443]\", \"lift_point\": \"[0.0435, 0.8443]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7086, 0.6197]\", \"lift_point\": \"[0.7086, 0.6197]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0435, 0.8443]\", \"lift_point\": \"[0.0435, 0.8443]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7086, 0.6197]\", \"lift_point\": \"[0.7086, 0.6197]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.9569]\", \"lift_point\": \"[0.7971, 0.9569]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0435, 0.8443]\", \"lift_point\": \"[0.0435, 0.8443]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7086, 0.6197]\", \"lift_point\": \"[0.7086, 0.6197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.9569]\", \"lift_point\": \"[0.7971, 0.9569]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7209, 0.9022]\", \"lift_point\": \"[0.7209, 0.9022]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6567, 0.8303]\", \"lift_point\": \"[0.6567, 0.8303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5128, 0.5664]\", \"lift_point\": \"[0.5128, 0.5664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0435, 0.8443]\", \"lift_point\": \"[0.0435, 0.8443]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7086, 0.6197]\", \"lift_point\": \"[0.7086, 0.6197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7971, 0.9569]\", \"lift_point\": \"[0.7971, 0.9569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7209, 0.9022]\", \"lift_point\": \"[0.7209, 0.9022]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25946_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.85, 0.6212]\", \"lift_point\": \"[0.85, 0.6212]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25974_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.85, 0.6212]\", \"lift_point\": \"[0.85, 0.6212]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6137]\", \"lift_point\": \"[0.0533, 0.6137]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25974_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.85, 0.6212]\", \"lift_point\": \"[0.85, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6137]\", \"lift_point\": \"[0.0533, 0.6137]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25974_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.85, 0.6212]\", \"lift_point\": \"[0.85, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6137]\", \"lift_point\": \"[0.0533, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25974_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.85, 0.6212]\", \"lift_point\": \"[0.85, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6137]\", \"lift_point\": \"[0.0533, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25974_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.85, 0.6212]\", \"lift_point\": \"[0.85, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6137]\", \"lift_point\": \"[0.0533, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.1941]\", \"lift_point\": \"[0.8064, 0.1941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25974_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.85, 0.6212]\", \"lift_point\": \"[0.85, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.6137]\", \"lift_point\": \"[0.0533, 0.6137]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Washington DC\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8064, 0.1941]\", \"lift_point\": \"[0.8064, 0.1941]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Washington DC?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25974_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8539, 0.651]\", \"lift_point\": \"[0.8539, 0.651]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25994_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8539, 0.651]\", \"lift_point\": \"[0.8539, 0.651]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.5533]\", \"lift_point\": \"[0.0449, 0.5533]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25994_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8539, 0.651]\", \"lift_point\": \"[0.8539, 0.651]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.5533]\", \"lift_point\": \"[0.0449, 0.5533]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25994_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8539, 0.651]\", \"lift_point\": \"[0.8539, 0.651]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.5533]\", \"lift_point\": \"[0.0449, 0.5533]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25994_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8539, 0.651]\", \"lift_point\": \"[0.8539, 0.651]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.5533]\", \"lift_point\": \"[0.0449, 0.5533]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3031, 0.3016]\", \"lift_point\": \"[0.3031, 0.3016]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25994_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8539, 0.651]\", \"lift_point\": \"[0.8539, 0.651]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.5533]\", \"lift_point\": \"[0.0449, 0.5533]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3031, 0.3016]\", \"lift_point\": \"[0.3031, 0.3016]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9173, 0.8478]\", \"lift_point\": \"[0.9173, 0.8478]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25994_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8539, 0.651]\", \"lift_point\": \"[0.8539, 0.651]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0449, 0.5533]\", \"lift_point\": \"[0.0449, 0.5533]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3031, 0.3016]\", \"lift_point\": \"[0.3031, 0.3016]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9173, 0.8478]\", \"lift_point\": \"[0.9173, 0.8478]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/25994_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in space science?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8471, 0.6275]\", \"lift_point\": \"[0.8471, 0.6275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26016_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8471, 0.6275]\", \"lift_point\": \"[0.8471, 0.6275]\", \"typed_text\": \"\" Goal: What's the latest news in space science?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.5966]\", \"lift_point\": \"[0.047, 0.5966]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26016_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8471, 0.6275]\", \"lift_point\": \"[0.8471, 0.6275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.5966]\", \"lift_point\": \"[0.047, 0.5966]\", \"typed_text\": \"\" Goal: What's the latest news in space science?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space science\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26016_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8471, 0.6275]\", \"lift_point\": \"[0.8471, 0.6275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.5966]\", \"lift_point\": \"[0.047, 0.5966]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space science\" Goal: What's the latest news in space science?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26016_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8471, 0.6275]\", \"lift_point\": \"[0.8471, 0.6275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.5966]\", \"lift_point\": \"[0.047, 0.5966]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space science\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in space science?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2036, 0.11]\", \"lift_point\": \"[0.2036, 0.11]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26016_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8471, 0.6275]\", \"lift_point\": \"[0.8471, 0.6275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.5966]\", \"lift_point\": \"[0.047, 0.5966]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space science\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2036, 0.11]\", \"lift_point\": \"[0.2036, 0.11]\", \"typed_text\": \"\" Goal: What's the latest news in space science?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26016_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26023_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8509, 0.6368]\", \"lift_point\": \"[0.8509, 0.6368]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26023_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8509, 0.6368]\", \"lift_point\": \"[0.8509, 0.6368]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5934]\", \"lift_point\": \"[0.0503, 0.5934]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26023_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8509, 0.6368]\", \"lift_point\": \"[0.8509, 0.6368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5934]\", \"lift_point\": \"[0.0503, 0.5934]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Chinese Yuan\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26023_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8509, 0.6368]\", \"lift_point\": \"[0.8509, 0.6368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5934]\", \"lift_point\": \"[0.0503, 0.5934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Chinese Yuan\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26023_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8509, 0.6368]\", \"lift_point\": \"[0.8509, 0.6368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5934]\", \"lift_point\": \"[0.0503, 0.5934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Chinese Yuan\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26023_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8509, 0.6368]\", \"lift_point\": \"[0.8509, 0.6368]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.5934]\", \"lift_point\": \"[0.0503, 0.5934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Chinese Yuan\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26023_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is the sun?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8455, 0.6212]\", \"lift_point\": \"[0.8455, 0.6212]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26063_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8455, 0.6212]\", \"lift_point\": \"[0.8455, 0.6212]\", \"typed_text\": \"\" Goal: How big is the sun?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.4637]\", \"lift_point\": \"[0.0479, 0.4637]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26063_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8455, 0.6212]\", \"lift_point\": \"[0.8455, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.4637]\", \"lift_point\": \"[0.0479, 0.4637]\", \"typed_text\": \"\" Goal: How big is the sun?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the sun\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26063_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8455, 0.6212]\", \"lift_point\": \"[0.8455, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.4637]\", \"lift_point\": \"[0.0479, 0.4637]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the sun\" Goal: How big is the sun?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26063_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8455, 0.6212]\", \"lift_point\": \"[0.8455, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.4637]\", \"lift_point\": \"[0.0479, 0.4637]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the sun\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is the sun?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.11, 0.2895]\", \"lift_point\": \"[0.11, 0.2895]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26063_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8455, 0.6212]\", \"lift_point\": \"[0.8455, 0.6212]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.4637]\", \"lift_point\": \"[0.0479, 0.4637]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the sun\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.11, 0.2895]\", \"lift_point\": \"[0.11, 0.2895]\", \"typed_text\": \"\" Goal: How big is the sun?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26063_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26095_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8811, 0.6426]\", \"lift_point\": \"[0.8811, 0.6426]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26095_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8811, 0.6426]\", \"lift_point\": \"[0.8811, 0.6426]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Atlanta\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26095_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8811, 0.6426]\", \"lift_point\": \"[0.8811, 0.6426]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Atlanta\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1256, 0.4002]\", \"lift_point\": \"[0.1256, 0.4002]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26095_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8811, 0.6426]\", \"lift_point\": \"[0.8811, 0.6426]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1256, 0.4002]\", \"lift_point\": \"[0.1256, 0.4002]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.4643]\", \"lift_point\": \"[0.3034, 0.4643]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26095_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8811, 0.6426]\", \"lift_point\": \"[0.8811, 0.6426]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1256, 0.4002]\", \"lift_point\": \"[0.1256, 0.4002]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.4643]\", \"lift_point\": \"[0.3034, 0.4643]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26095_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8826, 0.5338]\", \"lift_point\": \"[0.8826, 0.5338]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26122_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8826, 0.5338]\", \"lift_point\": \"[0.8826, 0.5338]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26122_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8826, 0.5338]\", \"lift_point\": \"[0.8826, 0.5338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1173, 0.4211]\", \"lift_point\": \"[0.1173, 0.4211]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26122_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8826, 0.5338]\", \"lift_point\": \"[0.8826, 0.5338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1173, 0.4211]\", \"lift_point\": \"[0.1173, 0.4211]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3009, 0.2795]\", \"lift_point\": \"[0.3009, 0.2795]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26122_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8826, 0.5338]\", \"lift_point\": \"[0.8826, 0.5338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1173, 0.4211]\", \"lift_point\": \"[0.1173, 0.4211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3009, 0.2795]\", \"lift_point\": \"[0.3009, 0.2795]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9254, 0.5018]\", \"lift_point\": \"[0.9254, 0.5018]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26122_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8826, 0.5338]\", \"lift_point\": \"[0.8826, 0.5338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1173, 0.4211]\", \"lift_point\": \"[0.1173, 0.4211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3009, 0.2795]\", \"lift_point\": \"[0.3009, 0.2795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9254, 0.5018]\", \"lift_point\": \"[0.9254, 0.5018]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Tokyo>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26122_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26170_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5077]\", \"lift_point\": \"[0.8928, 0.5077]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26170_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5077]\", \"lift_point\": \"[0.8928, 0.5077]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26170_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5077]\", \"lift_point\": \"[0.8928, 0.5077]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.115, 0.3462]\", \"lift_point\": \"[0.115, 0.3462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26170_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5077]\", \"lift_point\": \"[0.8928, 0.5077]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.115, 0.3462]\", \"lift_point\": \"[0.115, 0.3462]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26170_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8731, 0.5541]\", \"lift_point\": \"[0.8731, 0.5541]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26176_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8731, 0.5541]\", \"lift_point\": \"[0.8731, 0.5541]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26176_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8731, 0.5541]\", \"lift_point\": \"[0.8731, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1158, 0.3802]\", \"lift_point\": \"[0.1158, 0.3802]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26176_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8731, 0.5541]\", \"lift_point\": \"[0.8731, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1158, 0.3802]\", \"lift_point\": \"[0.1158, 0.3802]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2789, 0.2266]\", \"lift_point\": \"[0.2789, 0.2266]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26176_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8731, 0.5541]\", \"lift_point\": \"[0.8731, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1158, 0.3802]\", \"lift_point\": \"[0.1158, 0.3802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2789, 0.2266]\", \"lift_point\": \"[0.2789, 0.2266]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.9566]\", \"lift_point\": \"[0.7963, 0.9566]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26176_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8731, 0.5541]\", \"lift_point\": \"[0.8731, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1158, 0.3802]\", \"lift_point\": \"[0.1158, 0.3802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2789, 0.2266]\", \"lift_point\": \"[0.2789, 0.2266]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.9566]\", \"lift_point\": \"[0.7963, 0.9566]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26176_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26235_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8914, 0.4885]\", \"lift_point\": \"[0.8914, 0.4885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26235_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8914, 0.4885]\", \"lift_point\": \"[0.8914, 0.4885]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26235_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8914, 0.4885]\", \"lift_point\": \"[0.8914, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2885, 0.3708]\", \"lift_point\": \"[0.2885, 0.3708]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26235_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8914, 0.4885]\", \"lift_point\": \"[0.8914, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2885, 0.3708]\", \"lift_point\": \"[0.2885, 0.3708]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4614, 0.2669]\", \"lift_point\": \"[0.4614, 0.2669]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26235_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8914, 0.4885]\", \"lift_point\": \"[0.8914, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2885, 0.3708]\", \"lift_point\": \"[0.2885, 0.3708]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4614, 0.2669]\", \"lift_point\": \"[0.4614, 0.2669]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8981, 0.8524]\", \"lift_point\": \"[0.8981, 0.8524]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26235_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8914, 0.4885]\", \"lift_point\": \"[0.8914, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2885, 0.3708]\", \"lift_point\": \"[0.2885, 0.3708]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4614, 0.2669]\", \"lift_point\": \"[0.4614, 0.2669]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8981, 0.8524]\", \"lift_point\": \"[0.8981, 0.8524]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26235_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8855, 0.5648]\", \"lift_point\": \"[0.8855, 0.5648]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26309_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8855, 0.5648]\", \"lift_point\": \"[0.8855, 0.5648]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26309_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8855, 0.5648]\", \"lift_point\": \"[0.8855, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.4815]\", \"lift_point\": \"[0.1172, 0.4815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26309_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8855, 0.5648]\", \"lift_point\": \"[0.8855, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.4815]\", \"lift_point\": \"[0.1172, 0.4815]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5059, 0.3782]\", \"lift_point\": \"[0.5059, 0.3782]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26309_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8855, 0.5648]\", \"lift_point\": \"[0.8855, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.4815]\", \"lift_point\": \"[0.1172, 0.4815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5059, 0.3782]\", \"lift_point\": \"[0.5059, 0.3782]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8942, 0.6991]\", \"lift_point\": \"[0.8942, 0.6991]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26309_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8855, 0.5648]\", \"lift_point\": \"[0.8855, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this week?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.4815]\", \"lift_point\": \"[0.1172, 0.4815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5059, 0.3782]\", \"lift_point\": \"[0.5059, 0.3782]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8942, 0.6991]\", \"lift_point\": \"[0.8942, 0.6991]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26309_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26321_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.519]\", \"lift_point\": \"[0.8867, 0.519]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26321_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.519]\", \"lift_point\": \"[0.8867, 0.519]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26321_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.519]\", \"lift_point\": \"[0.8867, 0.519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1068, 0.4057]\", \"lift_point\": \"[0.1068, 0.4057]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26321_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.519]\", \"lift_point\": \"[0.8867, 0.519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1068, 0.4057]\", \"lift_point\": \"[0.1068, 0.4057]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3741, 0.3787]\", \"lift_point\": \"[0.3741, 0.3787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26321_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.519]\", \"lift_point\": \"[0.8867, 0.519]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1068, 0.4057]\", \"lift_point\": \"[0.1068, 0.4057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3741, 0.3787]\", \"lift_point\": \"[0.3741, 0.3787]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26321_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5771]\", \"lift_point\": \"[0.8671, 0.5771]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26326_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5771]\", \"lift_point\": \"[0.8671, 0.5771]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Guyana?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26326_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5771]\", \"lift_point\": \"[0.8671, 0.5771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Guyana?\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1821, 0.242]\", \"lift_point\": \"[0.1821, 0.242]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26326_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5771]\", \"lift_point\": \"[0.8671, 0.5771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Guyana?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1821, 0.242]\", \"lift_point\": \"[0.1821, 0.242]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2904, 0.3624]\", \"lift_point\": \"[0.2904, 0.3624]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26326_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5771]\", \"lift_point\": \"[0.8671, 0.5771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Guyana?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1821, 0.242]\", \"lift_point\": \"[0.1821, 0.242]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2904, 0.3624]\", \"lift_point\": \"[0.2904, 0.3624]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26326_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6306, 0.1657]\", \"lift_point\": \"[0.6306, 0.1657]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6306, 0.1657]\", \"lift_point\": \"[0.6306, 0.1657]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6435, 0.1326]\", \"lift_point\": \"[0.6435, 0.1326]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6306, 0.1657]\", \"lift_point\": \"[0.6306, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6435, 0.1326]\", \"lift_point\": \"[0.6435, 0.1326]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1828, 0.25]\", \"lift_point\": \"[0.1828, 0.25]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6306, 0.1657]\", \"lift_point\": \"[0.6306, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6435, 0.1326]\", \"lift_point\": \"[0.6435, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1828, 0.25]\", \"lift_point\": \"[0.1828, 0.25]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9047]\", \"lift_point\": \"[0.0703, 0.9047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6306, 0.1657]\", \"lift_point\": \"[0.6306, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6435, 0.1326]\", \"lift_point\": \"[0.6435, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1828, 0.25]\", \"lift_point\": \"[0.1828, 0.25]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9047]\", \"lift_point\": \"[0.0703, 0.9047]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6306, 0.1657]\", \"lift_point\": \"[0.6306, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6435, 0.1326]\", \"lift_point\": \"[0.6435, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1828, 0.25]\", \"lift_point\": \"[0.1828, 0.25]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9047]\", \"lift_point\": \"[0.0703, 0.9047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1277, 0.2562]\", \"lift_point\": \"[0.1277, 0.2562]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6306, 0.1657]\", \"lift_point\": \"[0.6306, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6435, 0.1326]\", \"lift_point\": \"[0.6435, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1828, 0.25]\", \"lift_point\": \"[0.1828, 0.25]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.9047]\", \"lift_point\": \"[0.0703, 0.9047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1277, 0.2562]\", \"lift_point\": \"[0.1277, 0.2562]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26593_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.3738]\", \"lift_point\": \"[0.063, 0.3738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.3738]\", \"lift_point\": \"[0.063, 0.3738]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.3738]\", \"lift_point\": \"[0.063, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.3738]\", \"lift_point\": \"[0.063, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.266]\", \"lift_point\": \"[0.1219, 0.266]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.3738]\", \"lift_point\": \"[0.063, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.266]\", \"lift_point\": \"[0.1219, 0.266]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.3738]\", \"lift_point\": \"[0.063, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.266]\", \"lift_point\": \"[0.1219, 0.266]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7103, 0.5417]\", \"lift_point\": \"[0.7103, 0.5417]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7974, 0.7043]\", \"lift_point\": \"[0.7974, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.063, 0.3738]\", \"lift_point\": \"[0.063, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Five Guys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.266]\", \"lift_point\": \"[0.1219, 0.266]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7103, 0.5417]\", \"lift_point\": \"[0.7103, 0.5417]\", \"typed_text\": \"\" Goal: What's on the menu at Five Guys?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26633_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6984]\", \"lift_point\": \"[0.7898, 0.6984]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26653_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6984]\", \"lift_point\": \"[0.7898, 0.6984]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.2489]\", \"lift_point\": \"[0.0478, 0.2489]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26653_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6984]\", \"lift_point\": \"[0.7898, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.2489]\", \"lift_point\": \"[0.0478, 0.2489]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pakistan\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26653_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6984]\", \"lift_point\": \"[0.7898, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.2489]\", \"lift_point\": \"[0.0478, 0.2489]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pakistan\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.6011]\", \"lift_point\": \"[0.1283, 0.6011]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26653_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6984]\", \"lift_point\": \"[0.7898, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.2489]\", \"lift_point\": \"[0.0478, 0.2489]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pakistan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.6011]\", \"lift_point\": \"[0.1283, 0.6011]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2222, 0.1763]\", \"lift_point\": \"[0.2222, 0.1763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26653_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7898, 0.6984]\", \"lift_point\": \"[0.7898, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.2489]\", \"lift_point\": \"[0.0478, 0.2489]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pakistan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.6011]\", \"lift_point\": \"[0.1283, 0.6011]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2222, 0.1763]\", \"lift_point\": \"[0.2222, 0.1763]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26653_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.6731]\", \"lift_point\": \"[0.7893, 0.6731]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26709_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.6731]\", \"lift_point\": \"[0.7893, 0.6731]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.5749]\", \"lift_point\": \"[0.0574, 0.5749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26709_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.6731]\", \"lift_point\": \"[0.7893, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.5749]\", \"lift_point\": \"[0.0574, 0.5749]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Sacramento\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26709_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.6731]\", \"lift_point\": \"[0.7893, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.5749]\", \"lift_point\": \"[0.0574, 0.5749]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Sacramento\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1244, 0.6074]\", \"lift_point\": \"[0.1244, 0.6074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26709_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.6731]\", \"lift_point\": \"[0.7893, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.5749]\", \"lift_point\": \"[0.0574, 0.5749]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Sacramento\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1244, 0.6074]\", \"lift_point\": \"[0.1244, 0.6074]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26709_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.6731]\", \"lift_point\": \"[0.7893, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.5749]\", \"lift_point\": \"[0.0574, 0.5749]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Sacramento\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1244, 0.6074]\", \"lift_point\": \"[0.1244, 0.6074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6056, 0.5318]\", \"lift_point\": \"[0.6056, 0.5318]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26709_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7893, 0.6731]\", \"lift_point\": \"[0.7893, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.5749]\", \"lift_point\": \"[0.0574, 0.5749]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Sacramento\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1244, 0.6074]\", \"lift_point\": \"[0.1244, 0.6074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6056, 0.5318]\", \"lift_point\": \"[0.6056, 0.5318]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26709_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a blue whale?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.5112]\", \"lift_point\": \"[0.3493, 0.5112]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.5112]\", \"lift_point\": \"[0.3493, 0.5112]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3763]\", \"lift_point\": \"[0.0583, 0.3763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.5112]\", \"lift_point\": \"[0.3493, 0.5112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3763]\", \"lift_point\": \"[0.0583, 0.3763]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9232]\", \"lift_point\": \"[0.0591, 0.9232]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.5112]\", \"lift_point\": \"[0.3493, 0.5112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3763]\", \"lift_point\": \"[0.0583, 0.3763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9232]\", \"lift_point\": \"[0.0591, 0.9232]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.5112]\", \"lift_point\": \"[0.3493, 0.5112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3763]\", \"lift_point\": \"[0.0583, 0.3763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9232]\", \"lift_point\": \"[0.0591, 0.9232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1409, 0.353]\", \"lift_point\": \"[0.1409, 0.353]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4096, 0.7499]\", \"lift_point\": \"[0.4096, 0.7499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.5112]\", \"lift_point\": \"[0.3493, 0.5112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3763]\", \"lift_point\": \"[0.0583, 0.3763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9232]\", \"lift_point\": \"[0.0591, 0.9232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1409, 0.353]\", \"lift_point\": \"[0.1409, 0.353]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/26875_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2697, 0.5114]\", \"lift_point\": \"[0.2697, 0.5114]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6548, 0.5648]\", \"lift_point\": \"[0.6548, 0.5648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0985, 0.4286]\", \"lift_point\": \"[0.0985, 0.4286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6296, 0.6156]\", \"lift_point\": \"[0.6296, 0.6156]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6296, 0.6156]\", \"lift_point\": \"[0.6296, 0.6156]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6296, 0.6156]\", \"lift_point\": \"[0.6296, 0.6156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8198, 0.4497]\", \"lift_point\": \"[0.8198, 0.4497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6296, 0.6156]\", \"lift_point\": \"[0.6296, 0.6156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8198, 0.4497]\", \"lift_point\": \"[0.8198, 0.4497]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6296, 0.6156]\", \"lift_point\": \"[0.6296, 0.6156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8198, 0.4497]\", \"lift_point\": \"[0.8198, 0.4497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6296, 0.6156]\", \"lift_point\": \"[0.6296, 0.6156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8198, 0.4497]\", \"lift_point\": \"[0.8198, 0.4497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5797, 0.5502]\", \"lift_point\": \"[0.5797, 0.5502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3872, 0.608]\", \"lift_point\": \"[0.3872, 0.608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6296, 0.6156]\", \"lift_point\": \"[0.6296, 0.6156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8198, 0.4497]\", \"lift_point\": \"[0.8198, 0.4497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5797, 0.5502]\", \"lift_point\": \"[0.5797, 0.5502]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27074_21.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6971, 0.5936]\", \"lift_point\": \"[0.6971, 0.5936]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.4903]\", \"lift_point\": \"[0.1128, 0.4903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from nyc to mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.4197]\", \"lift_point\": \"[0.543, 0.4197]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.4197]\", \"lift_point\": \"[0.543, 0.4197]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.583, 0.4119]\", \"lift_point\": \"[0.583, 0.4119]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.543, 0.4197]\", \"lift_point\": \"[0.543, 0.4197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.583, 0.4119]\", \"lift_point\": \"[0.583, 0.4119]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27231_19.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6848, 0.6442]\", \"lift_point\": \"[0.6848, 0.6442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1336, 0.445]\", \"lift_point\": \"[0.1336, 0.445]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3298, 0.5941]\", \"lift_point\": \"[0.3298, 0.5941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from sydney to buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3298, 0.5941]\", \"lift_point\": \"[0.3298, 0.5941]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27242_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.675, 0.6023]\", \"lift_point\": \"[0.675, 0.6023]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.675, 0.6023]\", \"lift_point\": \"[0.675, 0.6023]\", \"typed_text\": \"\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.4815]\", \"lift_point\": \"[0.1174, 0.4815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.675, 0.6023]\", \"lift_point\": \"[0.675, 0.6023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.4815]\", \"lift_point\": \"[0.1174, 0.4815]\", \"typed_text\": \"\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Nepal\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.675, 0.6023]\", \"lift_point\": \"[0.675, 0.6023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.4815]\", \"lift_point\": \"[0.1174, 0.4815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Nepal\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.675, 0.6023]\", \"lift_point\": \"[0.675, 0.6023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.4815]\", \"lift_point\": \"[0.1174, 0.4815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Nepal\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.675, 0.6023]\", \"lift_point\": \"[0.675, 0.6023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.4815]\", \"lift_point\": \"[0.1174, 0.4815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Nepal\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.447, 0.2565]\", \"lift_point\": \"[0.447, 0.2565]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.709, 0.6491]\", \"lift_point\": \"[0.709, 0.6491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.675, 0.6023]\", \"lift_point\": \"[0.675, 0.6023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.4815]\", \"lift_point\": \"[0.1174, 0.4815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Nepal\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.447, 0.2565]\", \"lift_point\": \"[0.447, 0.2565]\", \"typed_text\": \"\" Goal: What's the news in Nepal?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27326_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of the United States>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27447_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6689, 0.6297]\", \"lift_point\": \"[0.6689, 0.6297]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27447_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6689, 0.6297]\", \"lift_point\": \"[0.6689, 0.6297]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1155, 0.3948]\", \"lift_point\": \"[0.1155, 0.3948]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27447_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6689, 0.6297]\", \"lift_point\": \"[0.6689, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1155, 0.3948]\", \"lift_point\": \"[0.1155, 0.3948]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of United States\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27447_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6689, 0.6297]\", \"lift_point\": \"[0.6689, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1155, 0.3948]\", \"lift_point\": \"[0.1155, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of United States\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27447_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6689, 0.6297]\", \"lift_point\": \"[0.6689, 0.6297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1155, 0.3948]\", \"lift_point\": \"[0.1155, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of United States\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27447_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27574_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.657, 0.6143]\", \"lift_point\": \"[0.657, 0.6143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27574_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.657, 0.6143]\", \"lift_point\": \"[0.657, 0.6143]\", \"typed_text\": \"\" Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0934, 0.9228]\", \"lift_point\": \"[0.0934, 0.9228]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27574_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.657, 0.6143]\", \"lift_point\": \"[0.657, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0934, 0.9228]\", \"lift_point\": \"[0.0934, 0.9228]\", \"typed_text\": \"\" Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.2437]\", \"lift_point\": \"[0.3196, 0.2437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27574_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.657, 0.6143]\", \"lift_point\": \"[0.657, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0934, 0.9228]\", \"lift_point\": \"[0.0934, 0.9228]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.2437]\", \"lift_point\": \"[0.3196, 0.2437]\", \"typed_text\": \"\" Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27574_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6751, 0.6307]\", \"lift_point\": \"[0.6751, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6705, 0.6189]\", \"lift_point\": \"[0.6705, 0.6189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6886, 0.6201]\", \"lift_point\": \"[0.6886, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.5271]\", \"lift_point\": \"[0.1264, 0.5271]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2716, 0.4401]\", \"lift_point\": \"[0.2716, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6323, 0.3296]\", \"lift_point\": \"[0.6323, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6357, 0.3296]\", \"lift_point\": \"[0.6357, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8648, 0.5908]\", \"lift_point\": \"[0.8648, 0.5908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6208, 0.7456]\", \"lift_point\": \"[0.6208, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8546, 0.3449]\", \"lift_point\": \"[0.8546, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.9051]\", \"lift_point\": \"[0.1189, 0.9051]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3164, 0.0237]\", \"lift_point\": \"[0.3164, 0.0237]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9041, 0.6678]\", \"lift_point\": \"[0.9041, 0.6678]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4396, 0.1947]\", \"lift_point\": \"[0.4396, 0.1947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9041, 0.6678]\", \"lift_point\": \"[0.9041, 0.6678]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27613_32.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27753_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27753_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8074, 0.134]\", \"lift_point\": \"[0.8074, 0.134]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27753_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8074, 0.134]\", \"lift_point\": \"[0.8074, 0.134]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27753_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6621, 0.6229]\", \"lift_point\": \"[0.6621, 0.6229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27887_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6621, 0.6229]\", \"lift_point\": \"[0.6621, 0.6229]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0983, 0.4783]\", \"lift_point\": \"[0.0983, 0.4783]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27887_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6621, 0.6229]\", \"lift_point\": \"[0.6621, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0983, 0.4783]\", \"lift_point\": \"[0.0983, 0.4783]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of brazil\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27887_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6621, 0.6229]\", \"lift_point\": \"[0.6621, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0983, 0.4783]\", \"lift_point\": \"[0.0983, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of brazil\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27887_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6621, 0.6229]\", \"lift_point\": \"[0.6621, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0983, 0.4783]\", \"lift_point\": \"[0.0983, 0.4783]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of brazil\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27887_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6604, 0.614]\", \"lift_point\": \"[0.6604, 0.614]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.4787]\", \"lift_point\": \"[0.1315, 0.4787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0851, 0.4957]\", \"lift_point\": \"[0.0851, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital city of Italy\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6369, 0.5138]\", \"lift_point\": \"[0.6369, 0.5138]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6369, 0.5138]\", \"lift_point\": \"[0.6369, 0.5138]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/27975_20.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6673, 0.5903]\", \"lift_point\": \"[0.6673, 0.5903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5689, 0.5117]\", \"lift_point\": \"[0.5689, 0.5117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4767]\", \"lift_point\": \"[0.1147, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent in san francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5689, 0.5117]\", \"lift_point\": \"[0.5689, 0.5117]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28075_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of colombia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of colombia\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of colombia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of colombia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of colombia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of colombia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7693, 0.9368]\", \"lift_point\": \"[0.7693, 0.9368]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6609, 0.6025]\", \"lift_point\": \"[0.6609, 0.6025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.088, 0.4926]\", \"lift_point\": \"[0.088, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of colombia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7693, 0.9368]\", \"lift_point\": \"[0.7693, 0.9368]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28219_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Ecuador\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Ecuador\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Ecuador\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Ecuador\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Ecuador\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Ecuador\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2196, 0.2362]\", \"lift_point\": \"[0.2196, 0.2362]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.672, 0.6159]\", \"lift_point\": \"[0.672, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Ecuador\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2196, 0.2362]\", \"lift_point\": \"[0.2196, 0.2362]\", \"typed_text\": \"\" Goal: What's the news in Ecuador?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28468_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6684, 0.6136]\", \"lift_point\": \"[0.6684, 0.6136]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28499_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6684, 0.6136]\", \"lift_point\": \"[0.6684, 0.6136]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest technology news\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28499_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6684, 0.6136]\", \"lift_point\": \"[0.6684, 0.6136]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest technology news\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28499_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6684, 0.6136]\", \"lift_point\": \"[0.6684, 0.6136]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest technology news\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28499_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6684, 0.6136]\", \"lift_point\": \"[0.6684, 0.6136]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest technology news\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4153, 0.2788]\", \"lift_point\": \"[0.4153, 0.2788]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28499_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6684, 0.6136]\", \"lift_point\": \"[0.6684, 0.6136]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest technology news\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4153, 0.2788]\", \"lift_point\": \"[0.4153, 0.2788]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28499_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8662, 0.1249]\", \"lift_point\": \"[0.8662, 0.1249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3776, 0.5482]\", \"lift_point\": \"[0.3776, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6163, 0.5461]\", \"lift_point\": \"[0.6163, 0.5461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.07]\", \"lift_point\": \"[0.1094, 0.07]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.07]\", \"lift_point\": \"[0.1094, 0.07]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1207, 0.0852]\", \"lift_point\": \"[0.1207, 0.0852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6944, 0.4068]\", \"lift_point\": \"[0.6944, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6993, 0.5955]\", \"lift_point\": \"[0.6993, 0.5955]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4782, 0.8456]\", \"lift_point\": \"[0.4782, 0.8456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.07]\", \"lift_point\": \"[0.1094, 0.07]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1207, 0.0852]\", \"lift_point\": \"[0.1207, 0.0852]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/28603_26.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of France?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of France\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29182_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of France\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1799, 0.3409]\", \"lift_point\": \"[0.1799, 0.3409]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29182_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of France\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1799, 0.3409]\", \"lift_point\": \"[0.1799, 0.3409]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29182_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29203_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29203_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3682, 0.7279]\", \"lift_point\": \"[0.3682, 0.7279]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29203_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3682, 0.7279]\", \"lift_point\": \"[0.3682, 0.7279]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29203_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7932, 0.1401]\", \"lift_point\": \"[0.7932, 0.1401]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7932, 0.1401]\", \"lift_point\": \"[0.7932, 0.1401]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0787, 0.0942]\", \"lift_point\": \"[0.0787, 0.0942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7932, 0.1401]\", \"lift_point\": \"[0.7932, 0.1401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0787, 0.0942]\", \"lift_point\": \"[0.0787, 0.0942]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8713, 0.5122]\", \"lift_point\": \"[0.8713, 0.5122]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7932, 0.1401]\", \"lift_point\": \"[0.7932, 0.1401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0787, 0.0942]\", \"lift_point\": \"[0.0787, 0.0942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8713, 0.5122]\", \"lift_point\": \"[0.8713, 0.5122]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6238, 0.3816]\", \"lift_point\": \"[0.6238, 0.3816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7932, 0.1401]\", \"lift_point\": \"[0.7932, 0.1401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0787, 0.0942]\", \"lift_point\": \"[0.0787, 0.0942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8713, 0.5122]\", \"lift_point\": \"[0.8713, 0.5122]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6238, 0.3816]\", \"lift_point\": \"[0.6238, 0.3816]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4124, 0.4896]\", \"lift_point\": \"[0.4124, 0.4896]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7932, 0.1401]\", \"lift_point\": \"[0.7932, 0.1401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0787, 0.0942]\", \"lift_point\": \"[0.0787, 0.0942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8713, 0.5122]\", \"lift_point\": \"[0.8713, 0.5122]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6238, 0.3816]\", \"lift_point\": \"[0.6238, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4124, 0.4896]\", \"lift_point\": \"[0.4124, 0.4896]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7037, 0.7685]\", \"lift_point\": \"[0.7037, 0.7685]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8245, 0.4131]\", \"lift_point\": \"[0.8245, 0.4131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.1513]\", \"lift_point\": \"[0.7836, 0.1513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7932, 0.1401]\", \"lift_point\": \"[0.7932, 0.1401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0787, 0.0942]\", \"lift_point\": \"[0.0787, 0.0942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8713, 0.5122]\", \"lift_point\": \"[0.8713, 0.5122]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6238, 0.3816]\", \"lift_point\": \"[0.6238, 0.3816]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4124, 0.4896]\", \"lift_point\": \"[0.4124, 0.4896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7037, 0.7685]\", \"lift_point\": \"[0.7037, 0.7685]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29212_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3727, 0.3787]\", \"lift_point\": \"[0.3727, 0.3787]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3727, 0.3787]\", \"lift_point\": \"[0.3727, 0.3787]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3727, 0.3787]\", \"lift_point\": \"[0.3727, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.2186]\", \"lift_point\": \"[0.1519, 0.2186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3727, 0.3787]\", \"lift_point\": \"[0.3727, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2245, 0.6318]\", \"lift_point\": \"[0.2245, 0.6318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1952, 0.0642]\", \"lift_point\": \"[0.1952, 0.0642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3727, 0.3787]\", \"lift_point\": \"[0.3727, 0.3787]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4, 0.914]\", \"lift_point\": \"[0.4, 0.914]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29223_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3555, 0.5587]\", \"lift_point\": \"[0.3555, 0.5587]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3555, 0.5587]\", \"lift_point\": \"[0.3555, 0.5587]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3555, 0.5587]\", \"lift_point\": \"[0.3555, 0.5587]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3555, 0.5587]\", \"lift_point\": \"[0.3555, 0.5587]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Suriname?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3555, 0.5587]\", \"lift_point\": \"[0.3555, 0.5587]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Suriname?\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3555, 0.5587]\", \"lift_point\": \"[0.3555, 0.5587]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Suriname?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4069, 0.1882]\", \"lift_point\": \"[0.4069, 0.1882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3555, 0.5587]\", \"lift_point\": \"[0.3555, 0.5587]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.4558]\", \"lift_point\": \"[0.0639, 0.4558]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Suriname?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4069, 0.1882]\", \"lift_point\": \"[0.4069, 0.1882]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29267_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of sound?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.5149]\", \"lift_point\": \"[0.3643, 0.5149]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.5149]\", \"lift_point\": \"[0.3643, 0.5149]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.4319]\", \"lift_point\": \"[0.0756, 0.4319]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.5149]\", \"lift_point\": \"[0.3643, 0.5149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.4319]\", \"lift_point\": \"[0.0756, 0.4319]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4319]\", \"lift_point\": \"[0.0721, 0.4319]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.5149]\", \"lift_point\": \"[0.3643, 0.5149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.4319]\", \"lift_point\": \"[0.0756, 0.4319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4319]\", \"lift_point\": \"[0.0721, 0.4319]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of sound\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.5149]\", \"lift_point\": \"[0.3643, 0.5149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.4319]\", \"lift_point\": \"[0.0756, 0.4319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4319]\", \"lift_point\": \"[0.0721, 0.4319]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of sound\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3643, 0.5149]\", \"lift_point\": \"[0.3643, 0.5149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.4319]\", \"lift_point\": \"[0.0756, 0.4319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4319]\", \"lift_point\": \"[0.0721, 0.4319]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of sound\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of sound?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29393_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9485, 0.7779]\", \"lift_point\": \"[0.9485, 0.7779]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9485, 0.7779]\", \"lift_point\": \"[0.9485, 0.7779]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6718, 0.8082]\", \"lift_point\": \"[0.6718, 0.8082]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9485, 0.7779]\", \"lift_point\": \"[0.9485, 0.7779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6718, 0.8082]\", \"lift_point\": \"[0.6718, 0.8082]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5375, 0.6687]\", \"lift_point\": \"[0.5375, 0.6687]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9485, 0.7779]\", \"lift_point\": \"[0.9485, 0.7779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6718, 0.8082]\", \"lift_point\": \"[0.6718, 0.8082]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5375, 0.6687]\", \"lift_point\": \"[0.5375, 0.6687]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2518, 0.5285]\", \"lift_point\": \"[0.2518, 0.5285]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9485, 0.7779]\", \"lift_point\": \"[0.9485, 0.7779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6718, 0.8082]\", \"lift_point\": \"[0.6718, 0.8082]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5375, 0.6687]\", \"lift_point\": \"[0.5375, 0.6687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2518, 0.5285]\", \"lift_point\": \"[0.2518, 0.5285]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of switzerland ?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9485, 0.7779]\", \"lift_point\": \"[0.9485, 0.7779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6718, 0.8082]\", \"lift_point\": \"[0.6718, 0.8082]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5375, 0.6687]\", \"lift_point\": \"[0.5375, 0.6687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2518, 0.5285]\", \"lift_point\": \"[0.2518, 0.5285]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of switzerland?\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1132, 0.31]\", \"lift_point\": \"[0.1132, 0.31]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8279, 0.5825]\", \"lift_point\": \"[0.8279, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9338, 0.5386]\", \"lift_point\": \"[0.9338, 0.5386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9485, 0.7779]\", \"lift_point\": \"[0.9485, 0.7779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6718, 0.8082]\", \"lift_point\": \"[0.6718, 0.8082]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5375, 0.6687]\", \"lift_point\": \"[0.5375, 0.6687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2518, 0.5285]\", \"lift_point\": \"[0.2518, 0.5285]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of switzerland?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1132, 0.31]\", \"lift_point\": \"[0.1132, 0.31]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29441_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4388, 0.4604]\", \"lift_point\": \"[0.4388, 0.4604]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4388, 0.4604]\", \"lift_point\": \"[0.4388, 0.4604]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.135, 0.8465]\", \"lift_point\": \"[0.135, 0.8465]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4388, 0.4604]\", \"lift_point\": \"[0.4388, 0.4604]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.135, 0.8465]\", \"lift_point\": \"[0.135, 0.8465]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4388, 0.4604]\", \"lift_point\": \"[0.4388, 0.4604]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.135, 0.8465]\", \"lift_point\": \"[0.135, 0.8465]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3977, 0.5613]\", \"lift_point\": \"[0.3977, 0.5613]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4388, 0.4604]\", \"lift_point\": \"[0.4388, 0.4604]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.135, 0.8465]\", \"lift_point\": \"[0.135, 0.8465]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3977, 0.5613]\", \"lift_point\": \"[0.3977, 0.5613]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.593, 0.4368]\", \"lift_point\": \"[0.593, 0.4368]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8715, 0.6588]\", \"lift_point\": \"[0.8715, 0.6588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4388, 0.4604]\", \"lift_point\": \"[0.4388, 0.4604]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.135, 0.8465]\", \"lift_point\": \"[0.135, 0.8465]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3977, 0.5613]\", \"lift_point\": \"[0.3977, 0.5613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.593, 0.4368]\", \"lift_point\": \"[0.593, 0.4368]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29623_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6984]\", \"lift_point\": \"[0.8001, 0.6984]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6984]\", \"lift_point\": \"[0.8001, 0.6984]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9174, 0.4332]\", \"lift_point\": \"[0.9174, 0.4332]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6984]\", \"lift_point\": \"[0.8001, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9174, 0.4332]\", \"lift_point\": \"[0.9174, 0.4332]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9063, 0.8258]\", \"lift_point\": \"[0.9063, 0.8258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6984]\", \"lift_point\": \"[0.8001, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9174, 0.4332]\", \"lift_point\": \"[0.9174, 0.4332]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9063, 0.8258]\", \"lift_point\": \"[0.9063, 0.8258]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2569, 0.5632]\", \"lift_point\": \"[0.2569, 0.5632]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6984]\", \"lift_point\": \"[0.8001, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9174, 0.4332]\", \"lift_point\": \"[0.9174, 0.4332]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9063, 0.8258]\", \"lift_point\": \"[0.9063, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2569, 0.5632]\", \"lift_point\": \"[0.2569, 0.5632]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather like in Seoul\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6984]\", \"lift_point\": \"[0.8001, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9174, 0.4332]\", \"lift_point\": \"[0.9174, 0.4332]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9063, 0.8258]\", \"lift_point\": \"[0.9063, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2569, 0.5632]\", \"lift_point\": \"[0.2569, 0.5632]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather like in Seoul\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7952, 0.6984]\", \"lift_point\": \"[0.7952, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.6984]\", \"lift_point\": \"[0.8001, 0.6984]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9174, 0.4332]\", \"lift_point\": \"[0.9174, 0.4332]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9063, 0.8258]\", \"lift_point\": \"[0.9063, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2569, 0.5632]\", \"lift_point\": \"[0.2569, 0.5632]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather like in Seoul\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29692_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.6911]\", \"lift_point\": \"[0.7928, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8014, 0.6911]\", \"lift_point\": \"[0.8014, 0.6911]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2165, 0.5162]\", \"lift_point\": \"[0.2165, 0.5162]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.612]\", \"lift_point\": \"[0.0513, 0.612]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2165, 0.5162]\", \"lift_point\": \"[0.2165, 0.5162]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1361, 0.5295]\", \"lift_point\": \"[0.1361, 0.5295]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the latest video from the Huffington Post\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2165, 0.5162]\", \"lift_point\": \"[0.2165, 0.5162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1361, 0.5295]\", \"lift_point\": \"[0.1361, 0.5295]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1692, 0.5142]\", \"lift_point\": \"[0.1692, 0.5142]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.469, 0.2267]\", \"lift_point\": \"[0.469, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2165, 0.5162]\", \"lift_point\": \"[0.2165, 0.5162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1361, 0.5295]\", \"lift_point\": \"[0.1361, 0.5295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1692, 0.5142]\", \"lift_point\": \"[0.1692, 0.5142]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29719_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play some music on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.9146]\", \"lift_point\": \"[0.9018, 0.9146]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.9146]\", \"lift_point\": \"[0.9018, 0.9146]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"some music\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.9146]\", \"lift_point\": \"[0.9018, 0.9146]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"some music\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.9146]\", \"lift_point\": \"[0.9018, 0.9146]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"some music\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2655, 0.4144]\", \"lift_point\": \"[0.2655, 0.4144]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.8951]\", \"lift_point\": \"[0.7887, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.9146]\", \"lift_point\": \"[0.9018, 0.9146]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"some music\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2655, 0.4144]\", \"lift_point\": \"[0.2655, 0.4144]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1666, 0.5229]\", \"lift_point\": \"[0.1666, 0.5229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.8951]\", \"lift_point\": \"[0.7912, 0.8951]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.9146]\", \"lift_point\": \"[0.9018, 0.9146]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"some music\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2655, 0.4144]\", \"lift_point\": \"[0.2655, 0.4144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1666, 0.5229]\", \"lift_point\": \"[0.1666, 0.5229]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1697, 0.4941]\", \"lift_point\": \"[0.1697, 0.4941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0398, 0.7971]\", \"lift_point\": \"[0.0398, 0.7971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9018, 0.9146]\", \"lift_point\": \"[0.9018, 0.9146]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"some music\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2655, 0.4144]\", \"lift_point\": \"[0.2655, 0.4144]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1666, 0.5229]\", \"lift_point\": \"[0.1666, 0.5229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1697, 0.4941]\", \"lift_point\": \"[0.1697, 0.4941]\", \"typed_text\": \"\" Goal: Play some music on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29802_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6553, 0.6794]\", \"lift_point\": \"[0.6553, 0.6794]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6553, 0.6794]\", \"lift_point\": \"[0.6553, 0.6794]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.6953]\", \"lift_point\": \"[0.0587, 0.6953]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6553, 0.6794]\", \"lift_point\": \"[0.6553, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.6953]\", \"lift_point\": \"[0.0587, 0.6953]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.9241]\", \"lift_point\": \"[0.0538, 0.9241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6553, 0.6794]\", \"lift_point\": \"[0.6553, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.6953]\", \"lift_point\": \"[0.0587, 0.6953]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.9241]\", \"lift_point\": \"[0.0538, 0.9241]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6553, 0.6794]\", \"lift_point\": \"[0.6553, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.6953]\", \"lift_point\": \"[0.0587, 0.6953]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.9241]\", \"lift_point\": \"[0.0538, 0.9241]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Euro\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6553, 0.6794]\", \"lift_point\": \"[0.6553, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.6953]\", \"lift_point\": \"[0.0587, 0.6953]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.9241]\", \"lift_point\": \"[0.0538, 0.9241]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Euro\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1906, 0.4771]\", \"lift_point\": \"[0.1906, 0.4771]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6516, 0.6794]\", \"lift_point\": \"[0.6516, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6553, 0.6794]\", \"lift_point\": \"[0.6553, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0587, 0.6953]\", \"lift_point\": \"[0.0587, 0.6953]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.9241]\", \"lift_point\": \"[0.0538, 0.9241]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Euro\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1906, 0.4771]\", \"lift_point\": \"[0.1906, 0.4771]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29811_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of light?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.653, 0.6833]\", \"lift_point\": \"[0.653, 0.6833]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.653, 0.6833]\", \"lift_point\": \"[0.653, 0.6833]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5975]\", \"lift_point\": \"[0.0568, 0.5975]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.653, 0.6833]\", \"lift_point\": \"[0.653, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5975]\", \"lift_point\": \"[0.0568, 0.5975]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9429]\", \"lift_point\": \"[0.0549, 0.9429]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.653, 0.6833]\", \"lift_point\": \"[0.653, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5975]\", \"lift_point\": \"[0.0568, 0.5975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9429]\", \"lift_point\": \"[0.0549, 0.9429]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.653, 0.6833]\", \"lift_point\": \"[0.653, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5975]\", \"lift_point\": \"[0.0568, 0.5975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9429]\", \"lift_point\": \"[0.0549, 0.9429]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1167, 0.4031]\", \"lift_point\": \"[0.1167, 0.4031]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6506, 0.6833]\", \"lift_point\": \"[0.6506, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.653, 0.6833]\", \"lift_point\": \"[0.653, 0.6833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5975]\", \"lift_point\": \"[0.0568, 0.5975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0549, 0.9429]\", \"lift_point\": \"[0.0549, 0.9429]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of light\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1167, 0.4031]\", \"lift_point\": \"[0.1167, 0.4031]\", \"typed_text\": \"\" Goal: What is the speed of light?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/29819_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4378]\", \"lift_point\": \"[0.0578, 0.4378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4378]\", \"lift_point\": \"[0.0578, 0.4378]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US dollar exchange rate\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4378]\", \"lift_point\": \"[0.0578, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US dollar exchange rate\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.687]\", \"lift_point\": \"[0.7981, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4378]\", \"lift_point\": \"[0.0578, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US dollar exchange rate\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.4957]\", \"lift_point\": \"[0.332, 0.4957]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.713]\", \"lift_point\": \"[0.7885, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4378]\", \"lift_point\": \"[0.0578, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US dollar exchange rate\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.4957]\", \"lift_point\": \"[0.332, 0.4957]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6711]\", \"lift_point\": \"[0.7896, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4378]\", \"lift_point\": \"[0.0578, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US dollar exchange rate\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.4957]\", \"lift_point\": \"[0.332, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.242, 0.1486]\", \"lift_point\": \"[0.242, 0.1486]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.689]\", \"lift_point\": \"[0.8047, 0.689]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.7008]\", \"lift_point\": \"[0.7914, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.4378]\", \"lift_point\": \"[0.0578, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US dollar exchange rate\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.4957]\", \"lift_point\": \"[0.332, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.242, 0.1486]\", \"lift_point\": \"[0.242, 0.1486]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30076_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.9087]\", \"lift_point\": \"[0.1208, 0.9087]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.9087]\", \"lift_point\": \"[0.1208, 0.9087]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.0336]\", \"lift_point\": \"[0.0627, 0.0336]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.9087]\", \"lift_point\": \"[0.1208, 0.9087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.0336]\", \"lift_point\": \"[0.0627, 0.0336]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2841, 0.2791]\", \"lift_point\": \"[0.2841, 0.2791]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6838]\", \"lift_point\": \"[0.8049, 0.6838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.9087]\", \"lift_point\": \"[0.1208, 0.9087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.0336]\", \"lift_point\": \"[0.0627, 0.0336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2841, 0.2791]\", \"lift_point\": \"[0.2841, 0.2791]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Myanmar\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6976]\", \"lift_point\": \"[0.7931, 0.6976]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.9087]\", \"lift_point\": \"[0.1208, 0.9087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.0336]\", \"lift_point\": \"[0.0627, 0.0336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2841, 0.2791]\", \"lift_point\": \"[0.2841, 0.2791]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Myanmar\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.87]\", \"lift_point\": \"[0.0545, 0.87]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.9087]\", \"lift_point\": \"[0.1208, 0.9087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.0336]\", \"lift_point\": \"[0.0627, 0.0336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2841, 0.2791]\", \"lift_point\": \"[0.2841, 0.2791]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Myanmar\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2454, 0.1445]\", \"lift_point\": \"[0.2454, 0.1445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.9107]\", \"lift_point\": \"[0.1419, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.458, 0.9112]\", \"lift_point\": \"[0.458, 0.9112]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.9087]\", \"lift_point\": \"[0.1208, 0.9087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.0336]\", \"lift_point\": \"[0.0627, 0.0336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2841, 0.2791]\", \"lift_point\": \"[0.2841, 0.2791]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Myanmar\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2454, 0.1445]\", \"lift_point\": \"[0.2454, 0.1445]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30088_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30258_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3499, 0.878]\", \"lift_point\": \"[0.3499, 0.878]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30258_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3499, 0.878]\", \"lift_point\": \"[0.3499, 0.878]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.8742]\", \"lift_point\": \"[0.3683, 0.8742]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30258_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3499, 0.878]\", \"lift_point\": \"[0.3499, 0.878]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.8742]\", \"lift_point\": \"[0.3683, 0.8742]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30258_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3499, 0.878]\", \"lift_point\": \"[0.3499, 0.878]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.8742]\", \"lift_point\": \"[0.3683, 0.8742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30258_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3499, 0.878]\", \"lift_point\": \"[0.3499, 0.878]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.8742]\", \"lift_point\": \"[0.3683, 0.8742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6858, 0.8866]\", \"lift_point\": \"[0.6858, 0.8866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30258_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3499, 0.878]\", \"lift_point\": \"[0.3499, 0.878]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.8742]\", \"lift_point\": \"[0.3683, 0.8742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6858, 0.8866]\", \"lift_point\": \"[0.6858, 0.8866]\", \"typed_text\": \"\" Goal: Show me my notifications>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30258_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1232, 0.3528]\", \"lift_point\": \"[0.1232, 0.3528]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1232, 0.3528]\", \"lift_point\": \"[0.1232, 0.3528]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8609, 0.9214]\", \"lift_point\": \"[0.8609, 0.9214]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3576, 0.5059]\", \"lift_point\": \"[0.3576, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1232, 0.3528]\", \"lift_point\": \"[0.1232, 0.3528]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8609, 0.9214]\", \"lift_point\": \"[0.8609, 0.9214]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3492, 0.2792]\", \"lift_point\": \"[0.3492, 0.2792]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5227]\", \"lift_point\": \"[0.8887, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.8258]\", \"lift_point\": \"[0.891, 0.8258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2741, 0.5065]\", \"lift_point\": \"[0.2741, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Seattle?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1232, 0.3528]\", \"lift_point\": \"[0.1232, 0.3528]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8609, 0.9214]\", \"lift_point\": \"[0.8609, 0.9214]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3492, 0.2792]\", \"lift_point\": \"[0.3492, 0.2792]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30434_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.742]\", \"lift_point\": \"[0.1806, 0.742]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.742]\", \"lift_point\": \"[0.1806, 0.742]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.9366]\", \"lift_point\": \"[0.1092, 0.9366]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.742]\", \"lift_point\": \"[0.1806, 0.742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.9366]\", \"lift_point\": \"[0.1092, 0.9366]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best pizza restaurants on Maps\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.742]\", \"lift_point\": \"[0.1806, 0.742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.9366]\", \"lift_point\": \"[0.1092, 0.9366]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best pizza restaurants on Maps\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.742]\", \"lift_point\": \"[0.1806, 0.742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.9366]\", \"lift_point\": \"[0.1092, 0.9366]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best pizza restaurants on Maps\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1683, 0.2318]\", \"lift_point\": \"[0.1683, 0.2318]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.742]\", \"lift_point\": \"[0.1806, 0.742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.9366]\", \"lift_point\": \"[0.1092, 0.9366]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best pizza restaurants on Maps\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1683, 0.2318]\", \"lift_point\": \"[0.1683, 0.2318]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4786, 0.5535]\", \"lift_point\": \"[0.4786, 0.5535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3521, 0.4913]\", \"lift_point\": \"[0.3521, 0.4913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1806, 0.742]\", \"lift_point\": \"[0.1806, 0.742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.9366]\", \"lift_point\": \"[0.1092, 0.9366]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best pizza restaurants on Maps\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1683, 0.2318]\", \"lift_point\": \"[0.1683, 0.2318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4786, 0.5535]\", \"lift_point\": \"[0.4786, 0.5535]\", \"typed_text\": \"\" Goal: Search for the best pizza restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30477_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3598, 0.513]\", \"lift_point\": \"[0.3598, 0.513]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3598, 0.513]\", \"lift_point\": \"[0.3598, 0.513]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.713]\", \"lift_point\": \"[0.1847, 0.713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3598, 0.513]\", \"lift_point\": \"[0.3598, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.713]\", \"lift_point\": \"[0.1847, 0.713]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1061, 0.9273]\", \"lift_point\": \"[0.1061, 0.9273]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3598, 0.513]\", \"lift_point\": \"[0.3598, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.713]\", \"lift_point\": \"[0.1847, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1061, 0.9273]\", \"lift_point\": \"[0.1061, 0.9273]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my computer\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3598, 0.513]\", \"lift_point\": \"[0.3598, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.713]\", \"lift_point\": \"[0.1847, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1061, 0.9273]\", \"lift_point\": \"[0.1061, 0.9273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my computer\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2728, 0.2707]\", \"lift_point\": \"[0.2728, 0.2707]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3598, 0.513]\", \"lift_point\": \"[0.3598, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.713]\", \"lift_point\": \"[0.1847, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1061, 0.9273]\", \"lift_point\": \"[0.1061, 0.9273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my computer\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2728, 0.2707]\", \"lift_point\": \"[0.2728, 0.2707]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3256, 0.4458]\", \"lift_point\": \"[0.3256, 0.4458]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3598, 0.513]\", \"lift_point\": \"[0.3598, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.713]\", \"lift_point\": \"[0.1847, 0.713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1061, 0.9273]\", \"lift_point\": \"[0.1061, 0.9273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my computer\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2728, 0.2707]\", \"lift_point\": \"[0.2728, 0.2707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3256, 0.4458]\", \"lift_point\": \"[0.3256, 0.4458]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30495_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6961]\", \"lift_point\": \"[0.7774, 0.6961]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30574_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6961]\", \"lift_point\": \"[0.7774, 0.6961]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1855, 0.6422]\", \"lift_point\": \"[0.1855, 0.6422]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30574_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6961]\", \"lift_point\": \"[0.7774, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1855, 0.6422]\", \"lift_point\": \"[0.1855, 0.6422]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1185, 0.9216]\", \"lift_point\": \"[0.1185, 0.9216]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30574_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6961]\", \"lift_point\": \"[0.7774, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1855, 0.6422]\", \"lift_point\": \"[0.1855, 0.6422]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1185, 0.9216]\", \"lift_point\": \"[0.1185, 0.9216]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"iPhone 8 look like on eBay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30574_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6961]\", \"lift_point\": \"[0.7774, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1855, 0.6422]\", \"lift_point\": \"[0.1855, 0.6422]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1185, 0.9216]\", \"lift_point\": \"[0.1185, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"iPhone 8 look like on eBay?\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1754, 0.2773]\", \"lift_point\": \"[0.1754, 0.2773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30574_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7774, 0.6961]\", \"lift_point\": \"[0.7774, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1855, 0.6422]\", \"lift_point\": \"[0.1855, 0.6422]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1185, 0.9216]\", \"lift_point\": \"[0.1185, 0.9216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"iPhone 8 look like on eBay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1754, 0.2773]\", \"lift_point\": \"[0.1754, 0.2773]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30574_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30618_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30618_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Laos?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7732, 0.6982]\", \"lift_point\": \"[0.7732, 0.6982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7732, 0.6982]\", \"lift_point\": \"[0.7732, 0.6982]\", \"typed_text\": \"\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.7716]\", \"lift_point\": \"[0.1823, 0.7716]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7732, 0.6982]\", \"lift_point\": \"[0.7732, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.7716]\", \"lift_point\": \"[0.1823, 0.7716]\", \"typed_text\": \"\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.9265]\", \"lift_point\": \"[0.1099, 0.9265]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7732, 0.6982]\", \"lift_point\": \"[0.7732, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.7716]\", \"lift_point\": \"[0.1823, 0.7716]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.9265]\", \"lift_point\": \"[0.1099, 0.9265]\", \"typed_text\": \"\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Laos?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7732, 0.6982]\", \"lift_point\": \"[0.7732, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.7716]\", \"lift_point\": \"[0.1823, 0.7716]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.9265]\", \"lift_point\": \"[0.1099, 0.9265]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Laos?\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4281]\", \"lift_point\": \"[0.3812, 0.4281]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6013, 0.6804]\", \"lift_point\": \"[0.6013, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7732, 0.6982]\", \"lift_point\": \"[0.7732, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.7716]\", \"lift_point\": \"[0.1823, 0.7716]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.9265]\", \"lift_point\": \"[0.1099, 0.9265]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Laos?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4281]\", \"lift_point\": \"[0.3812, 0.4281]\", \"typed_text\": \"\" Goal: What's the news in Laos?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30666_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4671, 0.6866]\", \"lift_point\": \"[0.4671, 0.6866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4671, 0.6866]\", \"lift_point\": \"[0.4671, 0.6866]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1977, 0.5229]\", \"lift_point\": \"[0.1977, 0.5229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4671, 0.6866]\", \"lift_point\": \"[0.4671, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1977, 0.5229]\", \"lift_point\": \"[0.1977, 0.5229]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.919]\", \"lift_point\": \"[0.0591, 0.919]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4671, 0.6866]\", \"lift_point\": \"[0.4671, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1977, 0.5229]\", \"lift_point\": \"[0.1977, 0.5229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.919]\", \"lift_point\": \"[0.0591, 0.919]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4671, 0.6866]\", \"lift_point\": \"[0.4671, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1977, 0.5229]\", \"lift_point\": \"[0.1977, 0.5229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.919]\", \"lift_point\": \"[0.0591, 0.919]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico?\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4671, 0.6866]\", \"lift_point\": \"[0.4671, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1977, 0.5229]\", \"lift_point\": \"[0.1977, 0.5229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.919]\", \"lift_point\": \"[0.0591, 0.919]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5395, 0.3816]\", \"lift_point\": \"[0.5395, 0.3816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4671, 0.6866]\", \"lift_point\": \"[0.4671, 0.6866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1977, 0.5229]\", \"lift_point\": \"[0.1977, 0.5229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.919]\", \"lift_point\": \"[0.0591, 0.919]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5395, 0.3816]\", \"lift_point\": \"[0.5395, 0.3816]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/30916_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.8201]\", \"lift_point\": \"[0.0524, 0.8201]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.8201]\", \"lift_point\": \"[0.0524, 0.8201]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.9176]\", \"lift_point\": \"[0.057, 0.9176]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.8201]\", \"lift_point\": \"[0.0524, 0.8201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.9176]\", \"lift_point\": \"[0.057, 0.9176]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.8201]\", \"lift_point\": \"[0.0524, 0.8201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.9176]\", \"lift_point\": \"[0.057, 0.9176]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.8201]\", \"lift_point\": \"[0.0524, 0.8201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.9176]\", \"lift_point\": \"[0.057, 0.9176]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1227, 0.3341]\", \"lift_point\": \"[0.1227, 0.3341]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6075, 0.6952]\", \"lift_point\": \"[0.6075, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0524, 0.8201]\", \"lift_point\": \"[0.0524, 0.8201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.9176]\", \"lift_point\": \"[0.057, 0.9176]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1227, 0.3341]\", \"lift_point\": \"[0.1227, 0.3341]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31020_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0802, 0.0698]\", \"lift_point\": \"[0.0802, 0.0698]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0802, 0.0698]\", \"lift_point\": \"[0.0802, 0.0698]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8797, 0.5105]\", \"lift_point\": \"[0.8797, 0.5105]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0802, 0.0698]\", \"lift_point\": \"[0.0802, 0.0698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8797, 0.5105]\", \"lift_point\": \"[0.8797, 0.5105]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6133, 0.4004]\", \"lift_point\": \"[0.6133, 0.4004]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0802, 0.0698]\", \"lift_point\": \"[0.0802, 0.0698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8797, 0.5105]\", \"lift_point\": \"[0.8797, 0.5105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6133, 0.4004]\", \"lift_point\": \"[0.6133, 0.4004]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4131, 0.4954]\", \"lift_point\": \"[0.4131, 0.4954]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0802, 0.0698]\", \"lift_point\": \"[0.0802, 0.0698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8797, 0.5105]\", \"lift_point\": \"[0.8797, 0.5105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6133, 0.4004]\", \"lift_point\": \"[0.6133, 0.4004]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4131, 0.4954]\", \"lift_point\": \"[0.4131, 0.4954]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3334, 0.71]\", \"lift_point\": \"[0.3334, 0.71]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0802, 0.0698]\", \"lift_point\": \"[0.0802, 0.0698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8797, 0.5105]\", \"lift_point\": \"[0.8797, 0.5105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6133, 0.4004]\", \"lift_point\": \"[0.6133, 0.4004]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4131, 0.4954]\", \"lift_point\": \"[0.4131, 0.4954]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3334, 0.71]\", \"lift_point\": \"[0.3334, 0.71]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6824, 0.7429]\", \"lift_point\": \"[0.6824, 0.7429]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3779, 0.6628]\", \"lift_point\": \"[0.3779, 0.6628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0802, 0.0698]\", \"lift_point\": \"[0.0802, 0.0698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8797, 0.5105]\", \"lift_point\": \"[0.8797, 0.5105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6133, 0.4004]\", \"lift_point\": \"[0.6133, 0.4004]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4131, 0.4954]\", \"lift_point\": \"[0.4131, 0.4954]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3334, 0.71]\", \"lift_point\": \"[0.3334, 0.71]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6824, 0.7429]\", \"lift_point\": \"[0.6824, 0.7429]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31223_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.2882]\", \"lift_point\": \"[0.8906, 0.2882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31271_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.2882]\", \"lift_point\": \"[0.8906, 0.2882]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8964, 0.4679]\", \"lift_point\": \"[0.8964, 0.4679]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31271_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.2882]\", \"lift_point\": \"[0.8906, 0.2882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8964, 0.4679]\", \"lift_point\": \"[0.8964, 0.4679]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.2337]\", \"lift_point\": \"[0.0581, 0.2337]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31271_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.2882]\", \"lift_point\": \"[0.8906, 0.2882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8964, 0.4679]\", \"lift_point\": \"[0.8964, 0.4679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.2337]\", \"lift_point\": \"[0.0581, 0.2337]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31271_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.2882]\", \"lift_point\": \"[0.8906, 0.2882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8964, 0.4679]\", \"lift_point\": \"[0.8964, 0.4679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.2337]\", \"lift_point\": \"[0.0581, 0.2337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon?\" Goal: How big is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2506, 0.3563]\", \"lift_point\": \"[0.2506, 0.3563]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31271_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.2882]\", \"lift_point\": \"[0.8906, 0.2882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8964, 0.4679]\", \"lift_point\": \"[0.8964, 0.4679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.2337]\", \"lift_point\": \"[0.0581, 0.2337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the moon?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2506, 0.3563]\", \"lift_point\": \"[0.2506, 0.3563]\", \"typed_text\": \"\" Goal: How big is the moon?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31271_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.339]\", \"lift_point\": \"[0.1108, 0.339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.339]\", \"lift_point\": \"[0.1108, 0.339]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2375, 0.3558]\", \"lift_point\": \"[0.2375, 0.3558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.339]\", \"lift_point\": \"[0.1108, 0.339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2375, 0.3558]\", \"lift_point\": \"[0.2375, 0.3558]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1867, 0.335]\", \"lift_point\": \"[0.1867, 0.335]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7939, 0.8807]\", \"lift_point\": \"[0.7939, 0.8807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.339]\", \"lift_point\": \"[0.1108, 0.339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2375, 0.3558]\", \"lift_point\": \"[0.2375, 0.3558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1867, 0.335]\", \"lift_point\": \"[0.1867, 0.335]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.8517]\", \"lift_point\": \"[0.7817, 0.8517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.339]\", \"lift_point\": \"[0.1108, 0.339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2375, 0.3558]\", \"lift_point\": \"[0.2375, 0.3558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1867, 0.335]\", \"lift_point\": \"[0.1867, 0.335]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4122, 0.4985]\", \"lift_point\": \"[0.4122, 0.4985]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.8356]\", \"lift_point\": \"[0.0503, 0.8356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.339]\", \"lift_point\": \"[0.1108, 0.339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2375, 0.3558]\", \"lift_point\": \"[0.2375, 0.3558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1867, 0.335]\", \"lift_point\": \"[0.1867, 0.335]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4122, 0.4985]\", \"lift_point\": \"[0.4122, 0.4985]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2333, 0.8605]\", \"lift_point\": \"[0.2333, 0.8605]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4229]\", \"lift_point\": \"[0.0606, 0.4229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Taylor Swift\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.339]\", \"lift_point\": \"[0.1108, 0.339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2375, 0.3558]\", \"lift_point\": \"[0.2375, 0.3558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1867, 0.335]\", \"lift_point\": \"[0.1867, 0.335]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4122, 0.4985]\", \"lift_point\": \"[0.4122, 0.4985]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2333, 0.8605]\", \"lift_point\": \"[0.2333, 0.8605]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31284_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.2505]\", \"lift_point\": \"[0.0652, 0.2505]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31292_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.2505]\", \"lift_point\": \"[0.0652, 0.2505]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31292_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.2505]\", \"lift_point\": \"[0.0652, 0.2505]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4418]\", \"lift_point\": \"[0.1341, 0.4418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31292_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.2505]\", \"lift_point\": \"[0.0652, 0.2505]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4418]\", \"lift_point\": \"[0.1341, 0.4418]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31292_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Denver\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Denver\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n denver\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n denver\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1228, 0.7962]\", \"lift_point\": \"[0.1228, 0.7962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1228, 0.7962]\", \"lift_point\": \"[0.1228, 0.7962]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1228, 0.7962]\", \"lift_point\": \"[0.1228, 0.7962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1228, 0.7962]\", \"lift_point\": \"[0.1228, 0.7962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3453, 0.4593]\", \"lift_point\": \"[0.3453, 0.4593]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7911, 0.6743]\", \"lift_point\": \"[0.7911, 0.6743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.058, 0.4617]\", \"lift_point\": \"[0.058, 0.4617]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1228, 0.7962]\", \"lift_point\": \"[0.1228, 0.7962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3453, 0.4593]\", \"lift_point\": \"[0.3453, 0.4593]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31413_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rate against the Canadian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.6904]\", \"lift_point\": \"[0.7885, 0.6904]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31606_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.6904]\", \"lift_point\": \"[0.7885, 0.6904]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Canadian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9078, 0.4758]\", \"lift_point\": \"[0.9078, 0.4758]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31606_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.6904]\", \"lift_point\": \"[0.7885, 0.6904]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9078, 0.4758]\", \"lift_point\": \"[0.9078, 0.4758]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Canadian Dollar?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9064, 0.8726]\", \"lift_point\": \"[0.9064, 0.8726]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31606_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7885, 0.6904]\", \"lift_point\": \"[0.7885, 0.6904]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9078, 0.4758]\", \"lift_point\": \"[0.9078, 0.4758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9064, 0.8726]\", \"lift_point\": \"[0.9064, 0.8726]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Canadian Dollar?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31606_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31882_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8434, 0.4859]\", \"lift_point\": \"[0.8434, 0.4859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31882_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8434, 0.4859]\", \"lift_point\": \"[0.8434, 0.4859]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31882_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8434, 0.4859]\", \"lift_point\": \"[0.8434, 0.4859]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31882_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31887_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31887_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.831, 0.6115]\", \"lift_point\": \"[0.831, 0.6115]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31887_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.831, 0.6115]\", \"lift_point\": \"[0.831, 0.6115]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Philadelphia?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31887_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.831, 0.6115]\", \"lift_point\": \"[0.831, 0.6115]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Philadelphia?\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/31887_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0701, 0.8955]\", \"lift_point\": \"[0.0701, 0.8955]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0701, 0.8955]\", \"lift_point\": \"[0.0701, 0.8955]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Nordstrom\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0701, 0.8955]\", \"lift_point\": \"[0.0701, 0.8955]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Nordstrom\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4727]\", \"lift_point\": \"[0.1568, 0.4727]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0701, 0.8955]\", \"lift_point\": \"[0.0701, 0.8955]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Nordstrom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4727]\", \"lift_point\": \"[0.1568, 0.4727]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5489, 0.8252]\", \"lift_point\": \"[0.5489, 0.8252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0701, 0.8955]\", \"lift_point\": \"[0.0701, 0.8955]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Nordstrom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4727]\", \"lift_point\": \"[0.1568, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5489, 0.8252]\", \"lift_point\": \"[0.5489, 0.8252]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1427, 0.5831]\", \"lift_point\": \"[0.1427, 0.5831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0701, 0.8955]\", \"lift_point\": \"[0.0701, 0.8955]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Nordstrom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4727]\", \"lift_point\": \"[0.1568, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5489, 0.8252]\", \"lift_point\": \"[0.5489, 0.8252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1427, 0.5831]\", \"lift_point\": \"[0.1427, 0.5831]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.751, 0.8872]\", \"lift_point\": \"[0.751, 0.8872]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4826, 0.6852]\", \"lift_point\": \"[0.4826, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2261, 0.6257]\", \"lift_point\": \"[0.2261, 0.6257]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0701, 0.8955]\", \"lift_point\": \"[0.0701, 0.8955]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Nordstrom\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4727]\", \"lift_point\": \"[0.1568, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5489, 0.8252]\", \"lift_point\": \"[0.5489, 0.8252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1427, 0.5831]\", \"lift_point\": \"[0.1427, 0.5831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.751, 0.8872]\", \"lift_point\": \"[0.751, 0.8872]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32141_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this week?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6221, 0.9191]\", \"lift_point\": \"[0.6221, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4754, 0.7065]\", \"lift_point\": \"[0.4754, 0.7065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4754, 0.7065]\", \"lift_point\": \"[0.4754, 0.7065]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.7022]\", \"lift_point\": \"[0.1387, 0.7022]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4338, 0.3417]\", \"lift_point\": \"[0.4338, 0.3417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4754, 0.7065]\", \"lift_point\": \"[0.4754, 0.7065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.7022]\", \"lift_point\": \"[0.1387, 0.7022]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.907]\", \"lift_point\": \"[0.0551, 0.907]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4754, 0.7065]\", \"lift_point\": \"[0.4754, 0.7065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.7022]\", \"lift_point\": \"[0.1387, 0.7022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.907]\", \"lift_point\": \"[0.0551, 0.907]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4754, 0.7065]\", \"lift_point\": \"[0.4754, 0.7065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.7022]\", \"lift_point\": \"[0.1387, 0.7022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.907]\", \"lift_point\": \"[0.0551, 0.907]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1621, 0.4806]\", \"lift_point\": \"[0.1621, 0.4806]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4754, 0.7065]\", \"lift_point\": \"[0.4754, 0.7065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.7022]\", \"lift_point\": \"[0.1387, 0.7022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.907]\", \"lift_point\": \"[0.0551, 0.907]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1621, 0.4806]\", \"lift_point\": \"[0.1621, 0.4806]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1939, 0.2645]\", \"lift_point\": \"[0.1939, 0.2645]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4754, 0.7065]\", \"lift_point\": \"[0.4754, 0.7065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1387, 0.7022]\", \"lift_point\": \"[0.1387, 0.7022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.907]\", \"lift_point\": \"[0.0551, 0.907]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1621, 0.4806]\", \"lift_point\": \"[0.1621, 0.4806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1939, 0.2645]\", \"lift_point\": \"[0.1939, 0.2645]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32250_18.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.5312]\", \"lift_point\": \"[0.7799, 0.5312]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32413_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.5312]\", \"lift_point\": \"[0.7799, 0.5312]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" buy a nice beach chair\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32413_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.5312]\", \"lift_point\": \"[0.7799, 0.5312]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" buy a nice beach chair\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32413_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.5312]\", \"lift_point\": \"[0.7799, 0.5312]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" buy a nice beach chair\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2909, 0.6381]\", \"lift_point\": \"[0.2909, 0.6381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32413_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.5312]\", \"lift_point\": \"[0.7799, 0.5312]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" buy a nice beach chair\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2909, 0.6381]\", \"lift_point\": \"[0.2909, 0.6381]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach chair?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/32413_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33689_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7916, 0.5385]\", \"lift_point\": \"[0.7916, 0.5385]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33689_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7916, 0.5385]\", \"lift_point\": \"[0.7916, 0.5385]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4665]\", \"lift_point\": \"[0.0679, 0.4665]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33689_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7916, 0.5385]\", \"lift_point\": \"[0.7916, 0.5385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4665]\", \"lift_point\": \"[0.0679, 0.4665]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0769, 0.9198]\", \"lift_point\": \"[0.0769, 0.9198]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33689_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7916, 0.5385]\", \"lift_point\": \"[0.7916, 0.5385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4665]\", \"lift_point\": \"[0.0679, 0.4665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0769, 0.9198]\", \"lift_point\": \"[0.0769, 0.9198]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest Sprint Store\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33689_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7916, 0.5385]\", \"lift_point\": \"[0.7916, 0.5385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4665]\", \"lift_point\": \"[0.0679, 0.4665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0769, 0.9198]\", \"lift_point\": \"[0.0769, 0.9198]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest Sprint Store\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1658, 0.6097]\", \"lift_point\": \"[0.1658, 0.6097]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33689_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7916, 0.5385]\", \"lift_point\": \"[0.7916, 0.5385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4665]\", \"lift_point\": \"[0.0679, 0.4665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0769, 0.9198]\", \"lift_point\": \"[0.0769, 0.9198]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest Sprint Store\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1658, 0.6097]\", \"lift_point\": \"[0.1658, 0.6097]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33689_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33831_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.1105]\", \"lift_point\": \"[0.8042, 0.1105]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33831_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.1105]\", \"lift_point\": \"[0.8042, 0.1105]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4591]\", \"lift_point\": \"[0.0619, 0.4591]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33831_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.1105]\", \"lift_point\": \"[0.8042, 0.1105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4591]\", \"lift_point\": \"[0.0619, 0.4591]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of italy\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33831_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.1105]\", \"lift_point\": \"[0.8042, 0.1105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4591]\", \"lift_point\": \"[0.0619, 0.4591]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of italy\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1504, 0.4264]\", \"lift_point\": \"[0.1504, 0.4264]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33831_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8042, 0.1105]\", \"lift_point\": \"[0.8042, 0.1105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.4591]\", \"lift_point\": \"[0.0619, 0.4591]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of italy\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1504, 0.4264]\", \"lift_point\": \"[0.1504, 0.4264]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33831_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in China?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33860_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in China?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.5219]\", \"lift_point\": \"[0.8908, 0.5219]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33860_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.5219]\", \"lift_point\": \"[0.8908, 0.5219]\", \"typed_text\": \"\" Goal: What's the news in China?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in China\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33860_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.5219]\", \"lift_point\": \"[0.8908, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in China\" Goal: What's the news in China?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2318, 0.4508]\", \"lift_point\": \"[0.2318, 0.4508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33860_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.5219]\", \"lift_point\": \"[0.8908, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2318, 0.4508]\", \"lift_point\": \"[0.2318, 0.4508]\", \"typed_text\": \"\" Goal: What's the news in China?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33860_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.879, 0.4738]\", \"lift_point\": \"[0.879, 0.4738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33892_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.879, 0.4738]\", \"lift_point\": \"[0.879, 0.4738]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" a good restaurant in San Diego\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33892_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.879, 0.4738]\", \"lift_point\": \"[0.879, 0.4738]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" a good restaurant in San Diego\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.6906]\", \"lift_point\": \"[0.1221, 0.6906]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33892_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.879, 0.4738]\", \"lift_point\": \"[0.879, 0.4738]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" a good restaurant in San Diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.6906]\", \"lift_point\": \"[0.1221, 0.6906]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Diego?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/33892_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3531, 0.4998]\", \"lift_point\": \"[0.3531, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.373]\", \"lift_point\": \"[0.0679, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Houston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.43, 0.3015]\", \"lift_point\": \"[0.43, 0.3015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5927, 0.4049]\", \"lift_point\": \"[0.5927, 0.4049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5641, 0.4025]\", \"lift_point\": \"[0.5641, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8528, 0.3328]\", \"lift_point\": \"[0.8528, 0.3328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7562, 0.9092]\", \"lift_point\": \"[0.7562, 0.9092]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.3948]\", \"lift_point\": \"[0.8704, 0.3948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7562, 0.9092]\", \"lift_point\": \"[0.7562, 0.9092]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7627, 0.9254]\", \"lift_point\": \"[0.7627, 0.9254]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7562, 0.9092]\", \"lift_point\": \"[0.7562, 0.9092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7627, 0.9254]\", \"lift_point\": \"[0.7627, 0.9254]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3851, 0.781]\", \"lift_point\": \"[0.3851, 0.781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7562, 0.9092]\", \"lift_point\": \"[0.7562, 0.9092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7627, 0.9254]\", \"lift_point\": \"[0.7627, 0.9254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3851, 0.781]\", \"lift_point\": \"[0.3851, 0.781]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1594, 0.7356]\", \"lift_point\": \"[0.1594, 0.7356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7562, 0.9092]\", \"lift_point\": \"[0.7562, 0.9092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7627, 0.9254]\", \"lift_point\": \"[0.7627, 0.9254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3851, 0.781]\", \"lift_point\": \"[0.3851, 0.781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7513, 0.9036]\", \"lift_point\": \"[0.7513, 0.9036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7562, 0.9092]\", \"lift_point\": \"[0.7562, 0.9092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7627, 0.9254]\", \"lift_point\": \"[0.7627, 0.9254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3851, 0.781]\", \"lift_point\": \"[0.3851, 0.781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7513, 0.9036]\", \"lift_point\": \"[0.7513, 0.9036]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7526, 0.9309]\", \"lift_point\": \"[0.7526, 0.9309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3167, 0.6896]\", \"lift_point\": \"[0.3167, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7562, 0.9092]\", \"lift_point\": \"[0.7562, 0.9092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7627, 0.9254]\", \"lift_point\": \"[0.7627, 0.9254]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3851, 0.781]\", \"lift_point\": \"[0.3851, 0.781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7513, 0.9036]\", \"lift_point\": \"[0.7513, 0.9036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7526, 0.9309]\", \"lift_point\": \"[0.7526, 0.9309]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34141_35.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34230_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3624, 0.4989]\", \"lift_point\": \"[0.3624, 0.4989]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34230_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3624, 0.4989]\", \"lift_point\": \"[0.3624, 0.4989]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4767]\", \"lift_point\": \"[0.061, 0.4767]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34230_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3624, 0.4989]\", \"lift_point\": \"[0.3624, 0.4989]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4767]\", \"lift_point\": \"[0.061, 0.4767]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in orlando\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34230_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3624, 0.4989]\", \"lift_point\": \"[0.3624, 0.4989]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4767]\", \"lift_point\": \"[0.061, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in orlando\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34230_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3624, 0.4989]\", \"lift_point\": \"[0.3624, 0.4989]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.4767]\", \"lift_point\": \"[0.061, 0.4767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34230_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.4922]\", \"lift_point\": \"[0.3452, 0.4922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.4922]\", \"lift_point\": \"[0.3452, 0.4922]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4573]\", \"lift_point\": \"[0.0574, 0.4573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.4922]\", \"lift_point\": \"[0.3452, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4573]\", \"lift_point\": \"[0.0574, 0.4573]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.4922]\", \"lift_point\": \"[0.3452, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4573]\", \"lift_point\": \"[0.0574, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"+ Snow Blower?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.4922]\", \"lift_point\": \"[0.3452, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4573]\", \"lift_point\": \"[0.0574, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"+ Snow Blower?\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.4922]\", \"lift_point\": \"[0.3452, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0574, 0.4573]\", \"lift_point\": \"[0.0574, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"+ Snow Blower?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34246_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34559_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3224, 0.5171]\", \"lift_point\": \"[0.3224, 0.5171]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34559_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3224, 0.5171]\", \"lift_point\": \"[0.3224, 0.5171]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.5531]\", \"lift_point\": \"[0.0569, 0.5531]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34559_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3224, 0.5171]\", \"lift_point\": \"[0.3224, 0.5171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.5531]\", \"lift_point\": \"[0.0569, 0.5531]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34559_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3224, 0.5171]\", \"lift_point\": \"[0.3224, 0.5171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.5531]\", \"lift_point\": \"[0.0569, 0.5531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34559_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3224, 0.5171]\", \"lift_point\": \"[0.3224, 0.5171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.5531]\", \"lift_point\": \"[0.0569, 0.5531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34559_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4643, 0.7188]\", \"lift_point\": \"[0.4643, 0.7188]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4643, 0.7188]\", \"lift_point\": \"[0.4643, 0.7188]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4576, 0.6775]\", \"lift_point\": \"[0.4576, 0.6775]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4643, 0.7188]\", \"lift_point\": \"[0.4643, 0.7188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4576, 0.6775]\", \"lift_point\": \"[0.4576, 0.6775]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1919, 0.5291]\", \"lift_point\": \"[0.1919, 0.5291]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4643, 0.7188]\", \"lift_point\": \"[0.4643, 0.7188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4576, 0.6775]\", \"lift_point\": \"[0.4576, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1919, 0.5291]\", \"lift_point\": \"[0.1919, 0.5291]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9252]\", \"lift_point\": \"[0.0589, 0.9252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4643, 0.7188]\", \"lift_point\": \"[0.4643, 0.7188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4576, 0.6775]\", \"lift_point\": \"[0.4576, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1919, 0.5291]\", \"lift_point\": \"[0.1919, 0.5291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9252]\", \"lift_point\": \"[0.0589, 0.9252]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Panama\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4643, 0.7188]\", \"lift_point\": \"[0.4643, 0.7188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4576, 0.6775]\", \"lift_point\": \"[0.4576, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1919, 0.5291]\", \"lift_point\": \"[0.1919, 0.5291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9252]\", \"lift_point\": \"[0.0589, 0.9252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Panama\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4643, 0.7188]\", \"lift_point\": \"[0.4643, 0.7188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4576, 0.6775]\", \"lift_point\": \"[0.4576, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1919, 0.5291]\", \"lift_point\": \"[0.1919, 0.5291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9252]\", \"lift_point\": \"[0.0589, 0.9252]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Panama\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34567_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34588_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3248, 0.4949]\", \"lift_point\": \"[0.3248, 0.4949]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34588_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3248, 0.4949]\", \"lift_point\": \"[0.3248, 0.4949]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.7432]\", \"lift_point\": \"[0.1762, 0.7432]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34588_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3248, 0.4949]\", \"lift_point\": \"[0.3248, 0.4949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.7432]\", \"lift_point\": \"[0.1762, 0.7432]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1222, 0.9351]\", \"lift_point\": \"[0.1222, 0.9351]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34588_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3248, 0.4949]\", \"lift_point\": \"[0.3248, 0.4949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.7432]\", \"lift_point\": \"[0.1762, 0.7432]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1222, 0.9351]\", \"lift_point\": \"[0.1222, 0.9351]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Atlanta?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34588_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3248, 0.4949]\", \"lift_point\": \"[0.3248, 0.4949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.7432]\", \"lift_point\": \"[0.1762, 0.7432]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1222, 0.9351]\", \"lift_point\": \"[0.1222, 0.9351]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Atlanta?\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34588_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3248, 0.4949]\", \"lift_point\": \"[0.3248, 0.4949]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.7432]\", \"lift_point\": \"[0.1762, 0.7432]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1222, 0.9351]\", \"lift_point\": \"[0.1222, 0.9351]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Atlanta?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34588_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.4705]\", \"lift_point\": \"[0.0637, 0.4705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.4705]\", \"lift_point\": \"[0.0637, 0.4705]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.935]\", \"lift_point\": \"[0.0543, 0.935]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.4705]\", \"lift_point\": \"[0.0637, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.935]\", \"lift_point\": \"[0.0543, 0.935]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Install the Starbucks app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.4705]\", \"lift_point\": \"[0.0637, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.935]\", \"lift_point\": \"[0.0543, 0.935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Install the Starbucks app\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.4705]\", \"lift_point\": \"[0.0637, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.935]\", \"lift_point\": \"[0.0543, 0.935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Install the Starbucks app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1866, 0.3702]\", \"lift_point\": \"[0.1866, 0.3702]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.4705]\", \"lift_point\": \"[0.0637, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.935]\", \"lift_point\": \"[0.0543, 0.935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Install the Starbucks app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1866, 0.3702]\", \"lift_point\": \"[0.1866, 0.3702]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.5851]\", \"lift_point\": \"[0.2848, 0.5851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6001, 0.8815]\", \"lift_point\": \"[0.6001, 0.8815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6039, 0.8598]\", \"lift_point\": \"[0.6039, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0637, 0.4705]\", \"lift_point\": \"[0.0637, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.935]\", \"lift_point\": \"[0.0543, 0.935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Install the Starbucks app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1866, 0.3702]\", \"lift_point\": \"[0.1866, 0.3702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.5851]\", \"lift_point\": \"[0.2848, 0.5851]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34599_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.496]\", \"lift_point\": \"[0.3313, 0.496]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.496]\", \"lift_point\": \"[0.3313, 0.496]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1751, 0.5923]\", \"lift_point\": \"[0.1751, 0.5923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.496]\", \"lift_point\": \"[0.3313, 0.496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1751, 0.5923]\", \"lift_point\": \"[0.1751, 0.5923]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1069, 0.7598]\", \"lift_point\": \"[0.1069, 0.7598]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.496]\", \"lift_point\": \"[0.3313, 0.496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1751, 0.5923]\", \"lift_point\": \"[0.1751, 0.5923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1069, 0.7598]\", \"lift_point\": \"[0.1069, 0.7598]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1132, 0.9135]\", \"lift_point\": \"[0.1132, 0.9135]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.496]\", \"lift_point\": \"[0.3313, 0.496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1751, 0.5923]\", \"lift_point\": \"[0.1751, 0.5923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1069, 0.7598]\", \"lift_point\": \"[0.1069, 0.7598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1132, 0.9135]\", \"lift_point\": \"[0.1132, 0.9135]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Germany?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.496]\", \"lift_point\": \"[0.3313, 0.496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1751, 0.5923]\", \"lift_point\": \"[0.1751, 0.5923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1069, 0.7598]\", \"lift_point\": \"[0.1069, 0.7598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1132, 0.9135]\", \"lift_point\": \"[0.1132, 0.9135]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Germany?\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.496]\", \"lift_point\": \"[0.3313, 0.496]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1751, 0.5923]\", \"lift_point\": \"[0.1751, 0.5923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1069, 0.7598]\", \"lift_point\": \"[0.1069, 0.7598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1132, 0.9135]\", \"lift_point\": \"[0.1132, 0.9135]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Germany?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34631_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34686_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3222, 0.515]\", \"lift_point\": \"[0.3222, 0.515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34686_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3222, 0.515]\", \"lift_point\": \"[0.3222, 0.515]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0422, 0.3662]\", \"lift_point\": \"[0.0422, 0.3662]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34686_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3222, 0.515]\", \"lift_point\": \"[0.3222, 0.515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0422, 0.3662]\", \"lift_point\": \"[0.0422, 0.3662]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34686_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3222, 0.515]\", \"lift_point\": \"[0.3222, 0.515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0422, 0.3662]\", \"lift_point\": \"[0.0422, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34686_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3222, 0.515]\", \"lift_point\": \"[0.3222, 0.515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0422, 0.3662]\", \"lift_point\": \"[0.0422, 0.3662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34686_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34711_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.322, 0.3253]\", \"lift_point\": \"[0.322, 0.3253]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34711_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.322, 0.3253]\", \"lift_point\": \"[0.322, 0.3253]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8822, 0.4982]\", \"lift_point\": \"[0.8822, 0.4982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34711_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.322, 0.3253]\", \"lift_point\": \"[0.322, 0.3253]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8822, 0.4982]\", \"lift_point\": \"[0.8822, 0.4982]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7008, 0.544]\", \"lift_point\": \"[0.7008, 0.544]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34711_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.322, 0.3253]\", \"lift_point\": \"[0.322, 0.3253]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8822, 0.4982]\", \"lift_point\": \"[0.8822, 0.4982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7008, 0.544]\", \"lift_point\": \"[0.7008, 0.544]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34711_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34826_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.329, 0.5148]\", \"lift_point\": \"[0.329, 0.5148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34826_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.329, 0.5148]\", \"lift_point\": \"[0.329, 0.5148]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.053, 0.5106]\", \"lift_point\": \"[0.053, 0.5106]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34826_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.329, 0.5148]\", \"lift_point\": \"[0.329, 0.5148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.053, 0.5106]\", \"lift_point\": \"[0.053, 0.5106]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Walmart/\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34826_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.329, 0.5148]\", \"lift_point\": \"[0.329, 0.5148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.053, 0.5106]\", \"lift_point\": \"[0.053, 0.5106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Walmart/\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34826_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.329, 0.5148]\", \"lift_point\": \"[0.329, 0.5148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.053, 0.5106]\", \"lift_point\": \"[0.053, 0.5106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Walmart/\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34826_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34959_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34959_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3241, 0.4991]\", \"lift_point\": \"[0.3241, 0.4991]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34959_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3241, 0.4991]\", \"lift_point\": \"[0.3241, 0.4991]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0414, 0.4529]\", \"lift_point\": \"[0.0414, 0.4529]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34959_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3241, 0.4991]\", \"lift_point\": \"[0.3241, 0.4991]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0414, 0.4529]\", \"lift_point\": \"[0.0414, 0.4529]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a tiger?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34959_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3241, 0.4991]\", \"lift_point\": \"[0.3241, 0.4991]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0414, 0.4529]\", \"lift_point\": \"[0.0414, 0.4529]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a tiger?\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34959_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3241, 0.4991]\", \"lift_point\": \"[0.3241, 0.4991]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0414, 0.4529]\", \"lift_point\": \"[0.0414, 0.4529]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a tiger?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34959_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34971_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3171, 0.5037]\", \"lift_point\": \"[0.3171, 0.5037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34971_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3171, 0.5037]\", \"lift_point\": \"[0.3171, 0.5037]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0409, 0.5706]\", \"lift_point\": \"[0.0409, 0.5706]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34971_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3171, 0.5037]\", \"lift_point\": \"[0.3171, 0.5037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0409, 0.5706]\", \"lift_point\": \"[0.0409, 0.5706]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brail?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34971_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3171, 0.5037]\", \"lift_point\": \"[0.3171, 0.5037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0409, 0.5706]\", \"lift_point\": \"[0.0409, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brail?\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34971_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3171, 0.5037]\", \"lift_point\": \"[0.3171, 0.5037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0409, 0.5706]\", \"lift_point\": \"[0.0409, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brail?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34971_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4579]\", \"lift_point\": \"[0.3323, 0.4579]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4579]\", \"lift_point\": \"[0.3323, 0.4579]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4025]\", \"lift_point\": \"[0.584, 0.4025]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.1858]\", \"lift_point\": \"[0.1172, 0.1858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4579]\", \"lift_point\": \"[0.3323, 0.4579]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4025]\", \"lift_point\": \"[0.584, 0.4025]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.5293]\", \"lift_point\": \"[0.4178, 0.5293]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4579]\", \"lift_point\": \"[0.3323, 0.4579]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4025]\", \"lift_point\": \"[0.584, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.5293]\", \"lift_point\": \"[0.4178, 0.5293]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4182, 0.502]\", \"lift_point\": \"[0.4182, 0.502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4579]\", \"lift_point\": \"[0.3323, 0.4579]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4025]\", \"lift_point\": \"[0.584, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.5293]\", \"lift_point\": \"[0.4178, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4182, 0.502]\", \"lift_point\": \"[0.4182, 0.502]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3536, 0.69]\", \"lift_point\": \"[0.3536, 0.69]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4579]\", \"lift_point\": \"[0.3323, 0.4579]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4025]\", \"lift_point\": \"[0.584, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.5293]\", \"lift_point\": \"[0.4178, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4182, 0.502]\", \"lift_point\": \"[0.4182, 0.502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3536, 0.69]\", \"lift_point\": \"[0.3536, 0.69]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6697, 0.756]\", \"lift_point\": \"[0.6697, 0.756]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.32, 0.6924]\", \"lift_point\": \"[0.32, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5075]\", \"lift_point\": \"[0.8924, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4579]\", \"lift_point\": \"[0.3323, 0.4579]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4025]\", \"lift_point\": \"[0.584, 0.4025]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.5293]\", \"lift_point\": \"[0.4178, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4182, 0.502]\", \"lift_point\": \"[0.4182, 0.502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3536, 0.69]\", \"lift_point\": \"[0.3536, 0.69]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6697, 0.756]\", \"lift_point\": \"[0.6697, 0.756]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/34995_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3273, 0.513]\", \"lift_point\": \"[0.3273, 0.513]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4307]\", \"lift_point\": \"[0.0686, 0.4307]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6053, 0.4036]\", \"lift_point\": \"[0.6053, 0.4036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.3683]\", \"lift_point\": \"[0.0559, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6053, 0.4036]\", \"lift_point\": \"[0.6053, 0.4036]\", \"typed_text\": \"\" Goal: What's a good restaurant in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35093_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35278_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35278_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3232, 0.4702]\", \"lift_point\": \"[0.3232, 0.4702]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35278_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3232, 0.4702]\", \"lift_point\": \"[0.3232, 0.4702]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.5671]\", \"lift_point\": \"[0.0529, 0.5671]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35278_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3232, 0.4702]\", \"lift_point\": \"[0.3232, 0.4702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.5671]\", \"lift_point\": \"[0.0529, 0.5671]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Sydney?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35278_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3232, 0.4702]\", \"lift_point\": \"[0.3232, 0.4702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.5671]\", \"lift_point\": \"[0.0529, 0.5671]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Sydney?\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35278_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3232, 0.4702]\", \"lift_point\": \"[0.3232, 0.4702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0529, 0.5671]\", \"lift_point\": \"[0.0529, 0.5671]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Sydney?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35278_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35284_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.333, 0.5074]\", \"lift_point\": \"[0.333, 0.5074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35284_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.333, 0.5074]\", \"lift_point\": \"[0.333, 0.5074]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.5103]\", \"lift_point\": \"[0.0634, 0.5103]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35284_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.333, 0.5074]\", \"lift_point\": \"[0.333, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.5103]\", \"lift_point\": \"[0.0634, 0.5103]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35284_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.333, 0.5074]\", \"lift_point\": \"[0.333, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.5103]\", \"lift_point\": \"[0.0634, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35284_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.333, 0.5074]\", \"lift_point\": \"[0.333, 0.5074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.5103]\", \"lift_point\": \"[0.0634, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35284_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3203, 0.4723]\", \"lift_point\": \"[0.3203, 0.4723]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3203, 0.4723]\", \"lift_point\": \"[0.3203, 0.4723]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.5548]\", \"lift_point\": \"[0.0497, 0.5548]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3203, 0.4723]\", \"lift_point\": \"[0.3203, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.5548]\", \"lift_point\": \"[0.0497, 0.5548]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TLC TV\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3203, 0.4723]\", \"lift_point\": \"[0.3203, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.5548]\", \"lift_point\": \"[0.0497, 0.5548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TLC TV\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3203, 0.4723]\", \"lift_point\": \"[0.3203, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.5548]\", \"lift_point\": \"[0.0497, 0.5548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TLC TV\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3203, 0.4723]\", \"lift_point\": \"[0.3203, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0497, 0.5548]\", \"lift_point\": \"[0.0497, 0.5548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TLC TV\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"?\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35296_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35376_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35376_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.4995]\", \"lift_point\": \"[0.3196, 0.4995]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35376_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.4995]\", \"lift_point\": \"[0.3196, 0.4995]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5547]\", \"lift_point\": \"[0.0633, 0.5547]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35376_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.4995]\", \"lift_point\": \"[0.3196, 0.4995]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5547]\", \"lift_point\": \"[0.0633, 0.5547]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35376_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.4995]\", \"lift_point\": \"[0.3196, 0.4995]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5547]\", \"lift_point\": \"[0.0633, 0.5547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35376_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3196, 0.4995]\", \"lift_point\": \"[0.3196, 0.4995]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5547]\", \"lift_point\": \"[0.0633, 0.5547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35376_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the play store>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35420_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the play store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35420_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the play store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7473, 0.3229]\", \"lift_point\": \"[0.7473, 0.3229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35420_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7473, 0.3229]\", \"lift_point\": \"[0.7473, 0.3229]\", \"typed_text\": \"\" Goal: Open the play store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.6078]\", \"lift_point\": \"[0.046, 0.6078]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35420_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7473, 0.3229]\", \"lift_point\": \"[0.7473, 0.3229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.6078]\", \"lift_point\": \"[0.046, 0.6078]\", \"typed_text\": \"\" Goal: Open the play store>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.9373]\", \"lift_point\": \"[0.0571, 0.9373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35420_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7473, 0.3229]\", \"lift_point\": \"[0.7473, 0.3229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.6078]\", \"lift_point\": \"[0.046, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.9373]\", \"lift_point\": \"[0.0571, 0.9373]\", \"typed_text\": \"\" Goal: Open the play store>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.0764]\", \"lift_point\": \"[0.0473, 0.0764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35420_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7473, 0.3229]\", \"lift_point\": \"[0.7473, 0.3229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.6078]\", \"lift_point\": \"[0.046, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0571, 0.9373]\", \"lift_point\": \"[0.0571, 0.9373]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.0764]\", \"lift_point\": \"[0.0473, 0.0764]\", \"typed_text\": \"\" Goal: Open the play store>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35420_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35427_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6021, 0.5255]\", \"lift_point\": \"[0.6021, 0.5255]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35427_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6021, 0.5255]\", \"lift_point\": \"[0.6021, 0.5255]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.4697]\", \"lift_point\": \"[0.0694, 0.4697]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35427_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6021, 0.5255]\", \"lift_point\": \"[0.6021, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.4697]\", \"lift_point\": \"[0.0694, 0.4697]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9064]\", \"lift_point\": \"[0.0672, 0.9064]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35427_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6021, 0.5255]\", \"lift_point\": \"[0.6021, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.4697]\", \"lift_point\": \"[0.0694, 0.4697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9064]\", \"lift_point\": \"[0.0672, 0.9064]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated pizza restaurants on Maps\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35427_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6021, 0.5255]\", \"lift_point\": \"[0.6021, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.4697]\", \"lift_point\": \"[0.0694, 0.4697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9064]\", \"lift_point\": \"[0.0672, 0.9064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated pizza restaurants on Maps\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35427_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6021, 0.5255]\", \"lift_point\": \"[0.6021, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.4697]\", \"lift_point\": \"[0.0694, 0.4697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0672, 0.9064]\", \"lift_point\": \"[0.0672, 0.9064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated pizza restaurants on Maps\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for top rated pizza restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35427_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4574, 0.2937]\", \"lift_point\": \"[0.4574, 0.2937]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4574, 0.2937]\", \"lift_point\": \"[0.4574, 0.2937]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4574, 0.2937]\", \"lift_point\": \"[0.4574, 0.2937]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.33, 0.5079]\", \"lift_point\": \"[0.33, 0.5079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4574, 0.2937]\", \"lift_point\": \"[0.4574, 0.2937]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4371, 0.5148]\", \"lift_point\": \"[0.4371, 0.5148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.4272]\", \"lift_point\": \"[0.0703, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4574, 0.2937]\", \"lift_point\": \"[0.4574, 0.2937]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4371, 0.5148]\", \"lift_point\": \"[0.4371, 0.5148]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35463_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3307, 0.3501]\", \"lift_point\": \"[0.3307, 0.3501]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3289, 0.506]\", \"lift_point\": \"[0.3289, 0.506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.4976]\", \"lift_point\": \"[0.051, 0.4976]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Barcelona to Mexico city\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3307, 0.3501]\", \"lift_point\": \"[0.3307, 0.3501]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35545_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7778, 0.7176]\", \"lift_point\": \"[0.7778, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.6159]\", \"lift_point\": \"[0.872, 0.6159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8788, 0.859]\", \"lift_point\": \"[0.8788, 0.859]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.323, 0.3572]\", \"lift_point\": \"[0.323, 0.3572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what does the iphone 8 look like \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.4378]\", \"lift_point\": \"[0.1429, 0.4378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.9043]\", \"lift_point\": \"[0.8824, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5869, 0.4165]\", \"lift_point\": \"[0.5869, 0.4165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.3415]\", \"lift_point\": \"[0.5841, 0.3415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3656, 0.3271]\", \"lift_point\": \"[0.3656, 0.3271]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3656, 0.3271]\", \"lift_point\": \"[0.3656, 0.3271]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9059, 0.5166]\", \"lift_point\": \"[0.9059, 0.5166]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3656, 0.3271]\", \"lift_point\": \"[0.3656, 0.3271]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9059, 0.5166]\", \"lift_point\": \"[0.9059, 0.5166]\", \"typed_text\": \"\" Goal: What does the iPhone 8 look like?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35786_35.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Yahoo app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4017, 0.545]\", \"lift_point\": \"[0.4017, 0.545]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4017, 0.545]\", \"lift_point\": \"[0.4017, 0.545]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3566]\", \"lift_point\": \"[0.9011, 0.3566]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6568, 0.6713]\", \"lift_point\": \"[0.6568, 0.6713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4017, 0.545]\", \"lift_point\": \"[0.4017, 0.545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3566]\", \"lift_point\": \"[0.9011, 0.3566]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.2837]\", \"lift_point\": \"[0.8909, 0.2837]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.5378]\", \"lift_point\": \"[0.0633, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4017, 0.545]\", \"lift_point\": \"[0.4017, 0.545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3566]\", \"lift_point\": \"[0.9011, 0.3566]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.2837]\", \"lift_point\": \"[0.8909, 0.2837]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2941, 0.5099]\", \"lift_point\": \"[0.2941, 0.5099]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"yahoo app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4017, 0.545]\", \"lift_point\": \"[0.4017, 0.545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3566]\", \"lift_point\": \"[0.9011, 0.3566]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.2837]\", \"lift_point\": \"[0.8909, 0.2837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2941, 0.5099]\", \"lift_point\": \"[0.2941, 0.5099]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2899, 0.8218]\", \"lift_point\": \"[0.2899, 0.8218]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1534, 0.2986]\", \"lift_point\": \"[0.1534, 0.2986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4861, 0.1534]\", \"lift_point\": \"[0.4861, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4017, 0.545]\", \"lift_point\": \"[0.4017, 0.545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.3566]\", \"lift_point\": \"[0.9011, 0.3566]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.2837]\", \"lift_point\": \"[0.8909, 0.2837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2941, 0.5099]\", \"lift_point\": \"[0.2941, 0.5099]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2899, 0.8218]\", \"lift_point\": \"[0.2899, 0.8218]\", \"typed_text\": \"\" Goal: Install the Yahoo app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/35934_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0742, 0.4061]\", \"lift_point\": \"[0.0742, 0.4061]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36007_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0742, 0.4061]\", \"lift_point\": \"[0.0742, 0.4061]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from IGN\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36007_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0742, 0.4061]\", \"lift_point\": \"[0.0742, 0.4061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from IGN\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.4104]\", \"lift_point\": \"[0.1426, 0.4104]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36007_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0742, 0.4061]\", \"lift_point\": \"[0.0742, 0.4061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from IGN\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.4104]\", \"lift_point\": \"[0.1426, 0.4104]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4155, 0.1386]\", \"lift_point\": \"[0.4155, 0.1386]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36007_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0742, 0.4061]\", \"lift_point\": \"[0.0742, 0.4061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from IGN\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.4104]\", \"lift_point\": \"[0.1426, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4155, 0.1386]\", \"lift_point\": \"[0.4155, 0.1386]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.6415]\", \"lift_point\": \"[0.7387, 0.6415]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36007_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0742, 0.4061]\", \"lift_point\": \"[0.0742, 0.4061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from IGN\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.4104]\", \"lift_point\": \"[0.1426, 0.4104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4155, 0.1386]\", \"lift_point\": \"[0.4155, 0.1386]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7387, 0.6415]\", \"lift_point\": \"[0.7387, 0.6415]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36007_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from gamespot trailers\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from gamespot trailers\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from gamespot trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1416, 0.5087]\", \"lift_point\": \"[0.1416, 0.5087]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from gamespot trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1416, 0.5087]\", \"lift_point\": \"[0.1416, 0.5087]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.3653]\", \"lift_point\": \"[0.3782, 0.3653]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from gamespot trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1416, 0.5087]\", \"lift_point\": \"[0.1416, 0.5087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.3653]\", \"lift_point\": \"[0.3782, 0.3653]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from gamespot trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1416, 0.5087]\", \"lift_point\": \"[0.1416, 0.5087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.3653]\", \"lift_point\": \"[0.3782, 0.3653]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.3336]\", \"lift_point\": \"[0.0613, 0.3336]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7896, 0.6964]\", \"lift_point\": \"[0.7896, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3596]\", \"lift_point\": \"[0.0655, 0.3596]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from gamespot trailers\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1416, 0.5087]\", \"lift_point\": \"[0.1416, 0.5087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.3653]\", \"lift_point\": \"[0.3782, 0.3653]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.3336]\", \"lift_point\": \"[0.0613, 0.3336]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36149_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36549_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7744, 0.7101]\", \"lift_point\": \"[0.7744, 0.7101]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36549_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7744, 0.7101]\", \"lift_point\": \"[0.7744, 0.7101]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3938]\", \"lift_point\": \"[0.0649, 0.3938]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36549_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7744, 0.7101]\", \"lift_point\": \"[0.7744, 0.7101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3938]\", \"lift_point\": \"[0.0649, 0.3938]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get nearest t mobile store\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36549_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7744, 0.7101]\", \"lift_point\": \"[0.7744, 0.7101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3938]\", \"lift_point\": \"[0.0649, 0.3938]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get nearest t mobile store\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1422, 0.5736]\", \"lift_point\": \"[0.1422, 0.5736]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36549_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7744, 0.7101]\", \"lift_point\": \"[0.7744, 0.7101]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3938]\", \"lift_point\": \"[0.0649, 0.3938]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get nearest t mobile store\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1422, 0.5736]\", \"lift_point\": \"[0.1422, 0.5736]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36549_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7642, 0.6938]\", \"lift_point\": \"[0.7642, 0.6938]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36686_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7642, 0.6938]\", \"lift_point\": \"[0.7642, 0.6938]\", \"typed_text\": \"\" Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9402]\", \"lift_point\": \"[0.066, 0.9402]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36686_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7642, 0.6938]\", \"lift_point\": \"[0.7642, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9402]\", \"lift_point\": \"[0.066, 0.9402]\", \"typed_text\": \"\" Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1186, 0.6529]\", \"lift_point\": \"[0.1186, 0.6529]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36686_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7642, 0.6938]\", \"lift_point\": \"[0.7642, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.9402]\", \"lift_point\": \"[0.066, 0.9402]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1186, 0.6529]\", \"lift_point\": \"[0.1186, 0.6529]\", \"typed_text\": \"\" Goal: Open a new window in the chrome app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36686_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8823, 0.4497]\", \"lift_point\": \"[0.8823, 0.4497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36911_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8823, 0.4497]\", \"lift_point\": \"[0.8823, 0.4497]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36911_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8823, 0.4497]\", \"lift_point\": \"[0.8823, 0.4497]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi?\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1624, 0.3944]\", \"lift_point\": \"[0.1624, 0.3944]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36911_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8823, 0.4497]\", \"lift_point\": \"[0.8823, 0.4497]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Delhi?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1624, 0.3944]\", \"lift_point\": \"[0.1624, 0.3944]\", \"typed_text\": \"\" Goal: What's the weather like in Delhi?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/36911_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.1526]\", \"lift_point\": \"[0.7859, 0.1526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6656, 0.4608]\", \"lift_point\": \"[0.6656, 0.4608]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8291, 0.8973]\", \"lift_point\": \"[0.8291, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6656, 0.4608]\", \"lift_point\": \"[0.6656, 0.4608]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2277, 0.8749]\", \"lift_point\": \"[0.2277, 0.8749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.7975]\", \"lift_point\": \"[0.0688, 0.7975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6656, 0.4608]\", \"lift_point\": \"[0.6656, 0.4608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2277, 0.8749]\", \"lift_point\": \"[0.2277, 0.8749]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.5499]\", \"lift_point\": \"[0.1787, 0.5499]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Justin Bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6656, 0.4608]\", \"lift_point\": \"[0.6656, 0.4608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2277, 0.8749]\", \"lift_point\": \"[0.2277, 0.8749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.5499]\", \"lift_point\": \"[0.1787, 0.5499]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1679, 0.5235]\", \"lift_point\": \"[0.1679, 0.5235]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.4478]\", \"lift_point\": \"[0.1198, 0.4478]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6656, 0.4608]\", \"lift_point\": \"[0.6656, 0.4608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2277, 0.8749]\", \"lift_point\": \"[0.2277, 0.8749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.5499]\", \"lift_point\": \"[0.1787, 0.5499]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1679, 0.5235]\", \"lift_point\": \"[0.1679, 0.5235]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37070_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.6511]\", \"lift_point\": \"[0.8833, 0.6511]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37074_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.6511]\", \"lift_point\": \"[0.8833, 0.6511]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37074_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.6511]\", \"lift_point\": \"[0.8833, 0.6511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1585, 0.3225]\", \"lift_point\": \"[0.1585, 0.3225]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37074_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.6511]\", \"lift_point\": \"[0.8833, 0.6511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1585, 0.3225]\", \"lift_point\": \"[0.1585, 0.3225]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37074_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Sydney\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Sydney\" Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Sydney\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2061, 0.4438]\", \"lift_point\": \"[0.2061, 0.4438]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.4549]\", \"lift_point\": \"[0.8827, 0.4549]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Sydney\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2061, 0.4438]\", \"lift_point\": \"[0.2061, 0.4438]\", \"typed_text\": \"\" Goal: Search for hotels in Sydney>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37304_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4621]\", \"lift_point\": \"[0.8903, 0.4621]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4621]\", \"lift_point\": \"[0.8903, 0.4621]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9019, 0.4404]\", \"lift_point\": \"[0.9019, 0.4404]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4621]\", \"lift_point\": \"[0.8903, 0.4621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9019, 0.4404]\", \"lift_point\": \"[0.9019, 0.4404]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Pakistan\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4621]\", \"lift_point\": \"[0.8903, 0.4621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9019, 0.4404]\", \"lift_point\": \"[0.9019, 0.4404]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Pakistan\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4621]\", \"lift_point\": \"[0.8903, 0.4621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9019, 0.4404]\", \"lift_point\": \"[0.9019, 0.4404]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Pakistan\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1593, 0.4851]\", \"lift_point\": \"[0.1593, 0.4851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4621]\", \"lift_point\": \"[0.8903, 0.4621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9019, 0.4404]\", \"lift_point\": \"[0.9019, 0.4404]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Pakistan\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1593, 0.4851]\", \"lift_point\": \"[0.1593, 0.4851]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.211, 0.1925]\", \"lift_point\": \"[0.211, 0.1925]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4621]\", \"lift_point\": \"[0.8903, 0.4621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9019, 0.4404]\", \"lift_point\": \"[0.9019, 0.4404]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Pakistan\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1593, 0.4851]\", \"lift_point\": \"[0.1593, 0.4851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.211, 0.1925]\", \"lift_point\": \"[0.211, 0.1925]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37364_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6565]\", \"lift_point\": \"[0.4832, 0.6565]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6565]\", \"lift_point\": \"[0.4832, 0.6565]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2107, 0.2892]\", \"lift_point\": \"[0.2107, 0.2892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6565]\", \"lift_point\": \"[0.4832, 0.6565]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2107, 0.2892]\", \"lift_point\": \"[0.2107, 0.2892]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6565]\", \"lift_point\": \"[0.4832, 0.6565]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2107, 0.2892]\", \"lift_point\": \"[0.2107, 0.2892]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.2535]\", \"lift_point\": \"[0.1297, 0.2535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6565]\", \"lift_point\": \"[0.4832, 0.6565]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2107, 0.2892]\", \"lift_point\": \"[0.2107, 0.2892]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.2535]\", \"lift_point\": \"[0.1297, 0.2535]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3246, 0.3199]\", \"lift_point\": \"[0.3246, 0.3199]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4832, 0.6565]\", \"lift_point\": \"[0.4832, 0.6565]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2107, 0.2892]\", \"lift_point\": \"[0.2107, 0.2892]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.2535]\", \"lift_point\": \"[0.1297, 0.2535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3246, 0.3199]\", \"lift_point\": \"[0.3246, 0.3199]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37414_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4838, 0.6722]\", \"lift_point\": \"[0.4838, 0.6722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4838, 0.6722]\", \"lift_point\": \"[0.4838, 0.6722]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4841, 0.6812]\", \"lift_point\": \"[0.4841, 0.6812]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4838, 0.6722]\", \"lift_point\": \"[0.4838, 0.6722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4841, 0.6812]\", \"lift_point\": \"[0.4841, 0.6812]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1967, 0.4231]\", \"lift_point\": \"[0.1967, 0.4231]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4838, 0.6722]\", \"lift_point\": \"[0.4838, 0.6722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4841, 0.6812]\", \"lift_point\": \"[0.4841, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1967, 0.4231]\", \"lift_point\": \"[0.1967, 0.4231]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.915]\", \"lift_point\": \"[0.051, 0.915]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4838, 0.6722]\", \"lift_point\": \"[0.4838, 0.6722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4841, 0.6812]\", \"lift_point\": \"[0.4841, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1967, 0.4231]\", \"lift_point\": \"[0.1967, 0.4231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.915]\", \"lift_point\": \"[0.051, 0.915]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a tiger?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4838, 0.6722]\", \"lift_point\": \"[0.4838, 0.6722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4841, 0.6812]\", \"lift_point\": \"[0.4841, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1967, 0.4231]\", \"lift_point\": \"[0.1967, 0.4231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.915]\", \"lift_point\": \"[0.051, 0.915]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a tiger?\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.5324]\", \"lift_point\": \"[0.136, 0.5324]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4838, 0.6722]\", \"lift_point\": \"[0.4838, 0.6722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4841, 0.6812]\", \"lift_point\": \"[0.4841, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1967, 0.4231]\", \"lift_point\": \"[0.1967, 0.4231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.051, 0.915]\", \"lift_point\": \"[0.051, 0.915]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a tiger?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.5324]\", \"lift_point\": \"[0.136, 0.5324]\", \"typed_text\": \"\" Goal: What is the speed of a tiger?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37425_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6631]\", \"lift_point\": \"[0.7963, 0.6631]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37542_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6631]\", \"lift_point\": \"[0.7963, 0.6631]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.7379]\", \"lift_point\": \"[0.0561, 0.7379]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37542_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6631]\", \"lift_point\": \"[0.7963, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.7379]\", \"lift_point\": \"[0.0561, 0.7379]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37542_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6631]\", \"lift_point\": \"[0.7963, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.7379]\", \"lift_point\": \"[0.0561, 0.7379]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.3518]\", \"lift_point\": \"[0.1176, 0.3518]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37542_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6631]\", \"lift_point\": \"[0.7963, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.7379]\", \"lift_point\": \"[0.0561, 0.7379]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.3518]\", \"lift_point\": \"[0.1176, 0.3518]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2662, 0.2394]\", \"lift_point\": \"[0.2662, 0.2394]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37542_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7963, 0.6631]\", \"lift_point\": \"[0.7963, 0.6631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.7379]\", \"lift_point\": \"[0.0561, 0.7379]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.3518]\", \"lift_point\": \"[0.1176, 0.3518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2662, 0.2394]\", \"lift_point\": \"[0.2662, 0.2394]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37542_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.775, 0.4622]\", \"lift_point\": \"[0.775, 0.4622]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37545_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.775, 0.4622]\", \"lift_point\": \"[0.775, 0.4622]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.354]\", \"lift_point\": \"[0.0613, 0.354]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37545_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.775, 0.4622]\", \"lift_point\": \"[0.775, 0.4622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.354]\", \"lift_point\": \"[0.0613, 0.354]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37545_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.3667]\", \"lift_point\": \"[0.0606, 0.3667]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.3667]\", \"lift_point\": \"[0.0606, 0.3667]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.3667]\", \"lift_point\": \"[0.0606, 0.3667]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store?\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.9462]\", \"lift_point\": \"[0.1309, 0.9462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6865]\", \"lift_point\": \"[0.7854, 0.6865]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.691]\", \"lift_point\": \"[0.7887, 0.691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.3667]\", \"lift_point\": \"[0.0606, 0.3667]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.9462]\", \"lift_point\": \"[0.1309, 0.9462]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37570_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.1329]\", \"lift_point\": \"[0.0613, 0.1329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.1329]\", \"lift_point\": \"[0.0613, 0.1329]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.489]\", \"lift_point\": \"[0.8946, 0.489]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.1329]\", \"lift_point\": \"[0.0613, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.489]\", \"lift_point\": \"[0.8946, 0.489]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6363, 0.5188]\", \"lift_point\": \"[0.6363, 0.5188]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.1329]\", \"lift_point\": \"[0.0613, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.489]\", \"lift_point\": \"[0.8946, 0.489]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6363, 0.5188]\", \"lift_point\": \"[0.6363, 0.5188]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4066, 0.4998]\", \"lift_point\": \"[0.4066, 0.4998]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.1329]\", \"lift_point\": \"[0.0613, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.489]\", \"lift_point\": \"[0.8946, 0.489]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6363, 0.5188]\", \"lift_point\": \"[0.6363, 0.5188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4066, 0.4998]\", \"lift_point\": \"[0.4066, 0.4998]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3235, 0.7394]\", \"lift_point\": \"[0.3235, 0.7394]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.1329]\", \"lift_point\": \"[0.0613, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.489]\", \"lift_point\": \"[0.8946, 0.489]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6363, 0.5188]\", \"lift_point\": \"[0.6363, 0.5188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4066, 0.4998]\", \"lift_point\": \"[0.4066, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3235, 0.7394]\", \"lift_point\": \"[0.3235, 0.7394]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6897, 0.7722]\", \"lift_point\": \"[0.6897, 0.7722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.4874]\", \"lift_point\": \"[0.7687, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.1329]\", \"lift_point\": \"[0.0613, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.489]\", \"lift_point\": \"[0.8946, 0.489]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6363, 0.5188]\", \"lift_point\": \"[0.6363, 0.5188]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4066, 0.4998]\", \"lift_point\": \"[0.4066, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3235, 0.7394]\", \"lift_point\": \"[0.3235, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6897, 0.7722]\", \"lift_point\": \"[0.6897, 0.7722]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37578_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Play Music app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37657_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Music app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37657_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.845]\", \"lift_point\": \"[0.1121, 0.845]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3757, 0.6124]\", \"lift_point\": \"[0.3757, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.845]\", \"lift_point\": \"[0.1121, 0.845]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.6205]\", \"lift_point\": \"[0.3547, 0.6205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.845]\", \"lift_point\": \"[0.1121, 0.845]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.5857]\", \"lift_point\": \"[0.0715, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.845]\", \"lift_point\": \"[0.1121, 0.845]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.845]\", \"lift_point\": \"[0.1121, 0.845]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.845]\", \"lift_point\": \"[0.1121, 0.845]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.363]\", \"lift_point\": \"[0.3395, 0.363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0426, 0.819]\", \"lift_point\": \"[0.0426, 0.819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.845]\", \"lift_point\": \"[0.1121, 0.845]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.363]\", \"lift_point\": \"[0.3395, 0.363]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37713_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37815_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37815_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37815_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37889_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3387, 0.6066]\", \"lift_point\": \"[0.3387, 0.6066]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37889_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3387, 0.6066]\", \"lift_point\": \"[0.3387, 0.6066]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0732, 0.4089]\", \"lift_point\": \"[0.0732, 0.4089]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37889_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3387, 0.6066]\", \"lift_point\": \"[0.3387, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0732, 0.4089]\", \"lift_point\": \"[0.0732, 0.4089]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Sacramento?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37889_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3387, 0.6066]\", \"lift_point\": \"[0.3387, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0732, 0.4089]\", \"lift_point\": \"[0.0732, 0.4089]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Sacramento?\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37889_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3387, 0.6066]\", \"lift_point\": \"[0.3387, 0.6066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0732, 0.4089]\", \"lift_point\": \"[0.0732, 0.4089]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Sacramento?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Sacramento?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37889_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5701, 0.7052]\", \"lift_point\": \"[0.5701, 0.7052]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3326, 0.6003]\", \"lift_point\": \"[0.3326, 0.6003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5701, 0.7052]\", \"lift_point\": \"[0.5701, 0.7052]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.6266]\", \"lift_point\": \"[0.8123, 0.6266]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3293, 0.6078]\", \"lift_point\": \"[0.3293, 0.6078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5701, 0.7052]\", \"lift_point\": \"[0.5701, 0.7052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.6266]\", \"lift_point\": \"[0.8123, 0.6266]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.5081]\", \"lift_point\": \"[0.8047, 0.5081]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0663, 0.4031]\", \"lift_point\": \"[0.0663, 0.4031]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in space technology?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5701, 0.7052]\", \"lift_point\": \"[0.5701, 0.7052]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.6266]\", \"lift_point\": \"[0.8123, 0.6266]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.5081]\", \"lift_point\": \"[0.8047, 0.5081]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/37938_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38074_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8595, 0.4183]\", \"lift_point\": \"[0.8595, 0.4183]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38074_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8595, 0.4183]\", \"lift_point\": \"[0.8595, 0.4183]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38074_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8595, 0.4183]\", \"lift_point\": \"[0.8595, 0.4183]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.525]\", \"lift_point\": \"[0.1257, 0.525]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38074_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8595, 0.4183]\", \"lift_point\": \"[0.8595, 0.4183]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.525]\", \"lift_point\": \"[0.1257, 0.525]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38074_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8595, 0.4183]\", \"lift_point\": \"[0.8595, 0.4183]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.525]\", \"lift_point\": \"[0.1257, 0.525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6873, 0.4383]\", \"lift_point\": \"[0.6873, 0.4383]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38074_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8595, 0.4183]\", \"lift_point\": \"[0.8595, 0.4183]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1257, 0.525]\", \"lift_point\": \"[0.1257, 0.525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6873, 0.4383]\", \"lift_point\": \"[0.6873, 0.4383]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38074_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.8155]\", \"lift_point\": \"[0.0556, 0.8155]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.8155]\", \"lift_point\": \"[0.0556, 0.8155]\", \"typed_text\": \"\" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.7161]\", \"lift_point\": \"[0.0582, 0.7161]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.8155]\", \"lift_point\": \"[0.0556, 0.8155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.7161]\", \"lift_point\": \"[0.0582, 0.7161]\", \"typed_text\": \"\" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Drake \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.8155]\", \"lift_point\": \"[0.0556, 0.8155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.7161]\", \"lift_point\": \"[0.0582, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Drake \" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1139, 0.3459]\", \"lift_point\": \"[0.1139, 0.3459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.8155]\", \"lift_point\": \"[0.0556, 0.8155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.7161]\", \"lift_point\": \"[0.0582, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Drake \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1139, 0.3459]\", \"lift_point\": \"[0.1139, 0.3459]\", \"typed_text\": \"\" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3221, 0.4968]\", \"lift_point\": \"[0.3221, 0.4968]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.8155]\", \"lift_point\": \"[0.0556, 0.8155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.7161]\", \"lift_point\": \"[0.0582, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Drake \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1139, 0.3459]\", \"lift_point\": \"[0.1139, 0.3459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3221, 0.4968]\", \"lift_point\": \"[0.3221, 0.4968]\", \"typed_text\": \"\" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2111, 0.4884]\", \"lift_point\": \"[0.2111, 0.4884]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8719, 0.3538]\", \"lift_point\": \"[0.8719, 0.3538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4114, 0.5121]\", \"lift_point\": \"[0.4114, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.8155]\", \"lift_point\": \"[0.0556, 0.8155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.7161]\", \"lift_point\": \"[0.0582, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Play the new Drake \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1139, 0.3459]\", \"lift_point\": \"[0.1139, 0.3459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3221, 0.4968]\", \"lift_point\": \"[0.3221, 0.4968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2111, 0.4884]\", \"lift_point\": \"[0.2111, 0.4884]\", \"typed_text\": \"\" Goal: Play the new Drake video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38148_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8881, 0.4185]\", \"lift_point\": \"[0.8881, 0.4185]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38157_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8881, 0.4185]\", \"lift_point\": \"[0.8881, 0.4185]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38157_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8881, 0.4185]\", \"lift_point\": \"[0.8881, 0.4185]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City?\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4364, 0.5881]\", \"lift_point\": \"[0.4364, 0.5881]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38157_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8881, 0.4185]\", \"lift_point\": \"[0.8881, 0.4185]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4364, 0.5881]\", \"lift_point\": \"[0.4364, 0.5881]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38157_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay?\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2534, 0.3166]\", \"lift_point\": \"[0.2534, 0.3166]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2534, 0.3166]\", \"lift_point\": \"[0.2534, 0.3166]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2534, 0.3166]\", \"lift_point\": \"[0.2534, 0.3166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2534, 0.3166]\", \"lift_point\": \"[0.2534, 0.3166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7429, 0.2766]\", \"lift_point\": \"[0.7429, 0.2766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2534, 0.3166]\", \"lift_point\": \"[0.2534, 0.3166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7429, 0.2766]\", \"lift_point\": \"[0.7429, 0.2766]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9041, 0.5177]\", \"lift_point\": \"[0.9041, 0.5177]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.5023]\", \"lift_point\": \"[0.8773, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2534, 0.3166]\", \"lift_point\": \"[0.2534, 0.3166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7429, 0.2766]\", \"lift_point\": \"[0.7429, 0.2766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9041, 0.5177]\", \"lift_point\": \"[0.9041, 0.5177]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38190_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in London?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.409]\", \"lift_point\": \"[0.8974, 0.409]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38228_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.409]\", \"lift_point\": \"[0.8974, 0.409]\", \"typed_text\": \"\" Goal: What time is it in London?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in London?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38228_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.409]\", \"lift_point\": \"[0.8974, 0.409]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in London?\" Goal: What time is it in London?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.508]\", \"lift_point\": \"[0.1264, 0.508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38228_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.409]\", \"lift_point\": \"[0.8974, 0.409]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in London?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.508]\", \"lift_point\": \"[0.1264, 0.508]\", \"typed_text\": \"\" Goal: What time is it in London?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38228_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38357_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.6558]\", \"lift_point\": \"[0.8849, 0.6558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38357_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.6558]\", \"lift_point\": \"[0.8849, 0.6558]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38357_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.6558]\", \"lift_point\": \"[0.8849, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.6374]\", \"lift_point\": \"[0.1196, 0.6374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38357_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.6558]\", \"lift_point\": \"[0.8849, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.6374]\", \"lift_point\": \"[0.1196, 0.6374]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/38357_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39154_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3537, 0.8834]\", \"lift_point\": \"[0.3537, 0.8834]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39154_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3537, 0.8834]\", \"lift_point\": \"[0.3537, 0.8834]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39154_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39177_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.3826]\", \"lift_point\": \"[0.886, 0.3826]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39177_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.3826]\", \"lift_point\": \"[0.886, 0.3826]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8952, 0.4076]\", \"lift_point\": \"[0.8952, 0.4076]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39177_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.3826]\", \"lift_point\": \"[0.886, 0.3826]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8952, 0.4076]\", \"lift_point\": \"[0.8952, 0.4076]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5025]\", \"lift_point\": \"[0.8928, 0.5025]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39177_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.3826]\", \"lift_point\": \"[0.886, 0.3826]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8952, 0.4076]\", \"lift_point\": \"[0.8952, 0.4076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5025]\", \"lift_point\": \"[0.8928, 0.5025]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39177_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.3826]\", \"lift_point\": \"[0.886, 0.3826]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8952, 0.4076]\", \"lift_point\": \"[0.8952, 0.4076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5025]\", \"lift_point\": \"[0.8928, 0.5025]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in New York?\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39177_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.3826]\", \"lift_point\": \"[0.886, 0.3826]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8952, 0.4076]\", \"lift_point\": \"[0.8952, 0.4076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8928, 0.5025]\", \"lift_point\": \"[0.8928, 0.5025]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in New York?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39177_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39237_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.5067]\", \"lift_point\": \"[0.3197, 0.5067]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39237_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.5067]\", \"lift_point\": \"[0.3197, 0.5067]\", \"typed_text\": \"\" Goal: Search for hotels in Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.4793]\", \"lift_point\": \"[0.0471, 0.4793]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39237_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.5067]\", \"lift_point\": \"[0.3197, 0.5067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.4793]\", \"lift_point\": \"[0.0471, 0.4793]\", \"typed_text\": \"\" Goal: Search for hotels in Tokyo>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Tokyo\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39237_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.5067]\", \"lift_point\": \"[0.3197, 0.5067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.4793]\", \"lift_point\": \"[0.0471, 0.4793]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Tokyo\" Goal: Search for hotels in Tokyo>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39237_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.5067]\", \"lift_point\": \"[0.3197, 0.5067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.4793]\", \"lift_point\": \"[0.0471, 0.4793]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Tokyo\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Tokyo>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39237_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5143]\", \"lift_point\": \"[0.3556, 0.5143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5143]\", \"lift_point\": \"[0.3556, 0.5143]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.09, 0.3926]\", \"lift_point\": \"[0.09, 0.3926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5143]\", \"lift_point\": \"[0.3556, 0.5143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.09, 0.3926]\", \"lift_point\": \"[0.09, 0.3926]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.336]\", \"lift_point\": \"[0.0644, 0.336]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5143]\", \"lift_point\": \"[0.3556, 0.5143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.09, 0.3926]\", \"lift_point\": \"[0.09, 0.3926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.336]\", \"lift_point\": \"[0.0644, 0.336]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.9296]\", \"lift_point\": \"[0.0521, 0.9296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5143]\", \"lift_point\": \"[0.3556, 0.5143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.09, 0.3926]\", \"lift_point\": \"[0.09, 0.3926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.336]\", \"lift_point\": \"[0.0644, 0.336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.9296]\", \"lift_point\": \"[0.0521, 0.9296]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from Game Informer?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5143]\", \"lift_point\": \"[0.3556, 0.5143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.09, 0.3926]\", \"lift_point\": \"[0.09, 0.3926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.336]\", \"lift_point\": \"[0.0644, 0.336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.9296]\", \"lift_point\": \"[0.0521, 0.9296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from Game Informer?\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.5445]\", \"lift_point\": \"[0.1236, 0.5445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5143]\", \"lift_point\": \"[0.3556, 0.5143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.09, 0.3926]\", \"lift_point\": \"[0.09, 0.3926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0644, 0.336]\", \"lift_point\": \"[0.0644, 0.336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.9296]\", \"lift_point\": \"[0.0521, 0.9296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from Game Informer?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1236, 0.5445]\", \"lift_point\": \"[0.1236, 0.5445]\", \"typed_text\": \"\" Goal: What's the latest video from Game Informer?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39463_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3519, 0.4983]\", \"lift_point\": \"[0.3519, 0.4983]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3519, 0.4983]\", \"lift_point\": \"[0.3519, 0.4983]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3519, 0.4983]\", \"lift_point\": \"[0.3519, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.3925]\", \"lift_point\": \"[0.0562, 0.3925]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3519, 0.4983]\", \"lift_point\": \"[0.3519, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.3925]\", \"lift_point\": \"[0.0562, 0.3925]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.9378]\", \"lift_point\": \"[0.0533, 0.9378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3519, 0.4983]\", \"lift_point\": \"[0.3519, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.3925]\", \"lift_point\": \"[0.0562, 0.3925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.9378]\", \"lift_point\": \"[0.0533, 0.9378]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Rio de Janeiro?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3519, 0.4983]\", \"lift_point\": \"[0.3519, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.3925]\", \"lift_point\": \"[0.0562, 0.3925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.9378]\", \"lift_point\": \"[0.0533, 0.9378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Rio de Janeiro?\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.3516]\", \"lift_point\": \"[0.1189, 0.3516]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3519, 0.4983]\", \"lift_point\": \"[0.3519, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.3925]\", \"lift_point\": \"[0.0562, 0.3925]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0533, 0.9378]\", \"lift_point\": \"[0.0533, 0.9378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Rio de Janeiro?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.3516]\", \"lift_point\": \"[0.1189, 0.3516]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39477_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8073, 0.6794]\", \"lift_point\": \"[0.8073, 0.6794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.362, 0.5116]\", \"lift_point\": \"[0.362, 0.5116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0788, 0.5349]\", \"lift_point\": \"[0.0788, 0.5349]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"electronics store thats open now\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2153, 0.5998]\", \"lift_point\": \"[0.2153, 0.5998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4624]\", \"lift_point\": \"[0.1341, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2033, 0.5262]\", \"lift_point\": \"[0.2033, 0.5262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2643, 0.535]\", \"lift_point\": \"[0.2643, 0.535]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3254, 0.5145]\", \"lift_point\": \"[0.3254, 0.5145]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5419, 0.4345]\", \"lift_point\": \"[0.5419, 0.4345]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9138, 0.5418]\", \"lift_point\": \"[0.9138, 0.5418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.637, 0.9457]\", \"lift_point\": \"[0.637, 0.9457]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1772, 0.578]\", \"lift_point\": \"[0.1772, 0.578]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1772, 0.578]\", \"lift_point\": \"[0.1772, 0.578]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39632_41.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.6896]\", \"lift_point\": \"[0.7763, 0.6896]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.6896]\", \"lift_point\": \"[0.7763, 0.6896]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0438, 0.4508]\", \"lift_point\": \"[0.0438, 0.4508]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.6896]\", \"lift_point\": \"[0.7763, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0438, 0.4508]\", \"lift_point\": \"[0.0438, 0.4508]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.6896]\", \"lift_point\": \"[0.7763, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0438, 0.4508]\", \"lift_point\": \"[0.0438, 0.4508]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.1916]\", \"lift_point\": \"[0.1121, 0.1916]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.6896]\", \"lift_point\": \"[0.7763, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0438, 0.4508]\", \"lift_point\": \"[0.0438, 0.4508]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.1916]\", \"lift_point\": \"[0.1121, 0.1916]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6572, 0.2437]\", \"lift_point\": \"[0.6572, 0.2437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.6896]\", \"lift_point\": \"[0.7763, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0438, 0.4508]\", \"lift_point\": \"[0.0438, 0.4508]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.1916]\", \"lift_point\": \"[0.1121, 0.1916]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6572, 0.2437]\", \"lift_point\": \"[0.6572, 0.2437]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6587, 0.4894]\", \"lift_point\": \"[0.6587, 0.4894]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.6896]\", \"lift_point\": \"[0.7763, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0438, 0.4508]\", \"lift_point\": \"[0.0438, 0.4508]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.1916]\", \"lift_point\": \"[0.1121, 0.1916]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6572, 0.2437]\", \"lift_point\": \"[0.6572, 0.2437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6587, 0.4894]\", \"lift_point\": \"[0.6587, 0.4894]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39733_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cosmology news\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cosmology news\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1122, 0.1705]\", \"lift_point\": \"[0.1122, 0.1705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cosmology news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1122, 0.1705]\", \"lift_point\": \"[0.1122, 0.1705]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7796, 0.2768]\", \"lift_point\": \"[0.7796, 0.2768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cosmology news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1122, 0.1705]\", \"lift_point\": \"[0.1122, 0.1705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7796, 0.2768]\", \"lift_point\": \"[0.7796, 0.2768]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6401, 0.5209]\", \"lift_point\": \"[0.6401, 0.5209]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cosmology news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1122, 0.1705]\", \"lift_point\": \"[0.1122, 0.1705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7796, 0.2768]\", \"lift_point\": \"[0.7796, 0.2768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6401, 0.5209]\", \"lift_point\": \"[0.6401, 0.5209]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cosmology news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1122, 0.1705]\", \"lift_point\": \"[0.1122, 0.1705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7796, 0.2768]\", \"lift_point\": \"[0.7796, 0.2768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6401, 0.5209]\", \"lift_point\": \"[0.6401, 0.5209]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1678, 0.2926]\", \"lift_point\": \"[0.1678, 0.2926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7751, 0.6872]\", \"lift_point\": \"[0.7751, 0.6872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.4193]\", \"lift_point\": \"[0.0418, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cosmology news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1122, 0.1705]\", \"lift_point\": \"[0.1122, 0.1705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7796, 0.2768]\", \"lift_point\": \"[0.7796, 0.2768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6401, 0.5209]\", \"lift_point\": \"[0.6401, 0.5209]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1678, 0.2926]\", \"lift_point\": \"[0.1678, 0.2926]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39868_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2879]\", \"lift_point\": \"[0.1182, 0.2879]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6964]\", \"lift_point\": \"[0.7792, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2879]\", \"lift_point\": \"[0.1182, 0.2879]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0581, 0.8475]\", \"lift_point\": \"[0.0581, 0.8475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2879]\", \"lift_point\": \"[0.1182, 0.2879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7409, 0.3148]\", \"lift_point\": \"[0.7409, 0.3148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.048, 0.4408]\", \"lift_point\": \"[0.048, 0.4408]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2879]\", \"lift_point\": \"[0.1182, 0.2879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7409, 0.3148]\", \"lift_point\": \"[0.7409, 0.3148]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.9077]\", \"lift_point\": \"[0.0491, 0.9077]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1238, 0.2276]\", \"lift_point\": \"[0.1238, 0.2276]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2879]\", \"lift_point\": \"[0.1182, 0.2879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7409, 0.3148]\", \"lift_point\": \"[0.7409, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.9077]\", \"lift_point\": \"[0.0491, 0.9077]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6481, 0.7569]\", \"lift_point\": \"[0.6481, 0.7569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2879]\", \"lift_point\": \"[0.1182, 0.2879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7409, 0.3148]\", \"lift_point\": \"[0.7409, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.9077]\", \"lift_point\": \"[0.0491, 0.9077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.486, 0.494]\", \"lift_point\": \"[0.486, 0.494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.3995]\", \"lift_point\": \"[0.0457, 0.3995]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest grocery store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1182, 0.2879]\", \"lift_point\": \"[0.1182, 0.2879]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7409, 0.3148]\", \"lift_point\": \"[0.7409, 0.3148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0491, 0.9077]\", \"lift_point\": \"[0.0491, 0.9077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.486, 0.494]\", \"lift_point\": \"[0.486, 0.494]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/39969_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.097, 0.4882]\", \"lift_point\": \"[0.097, 0.4882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.097, 0.4882]\", \"lift_point\": \"[0.097, 0.4882]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0881, 0.9045]\", \"lift_point\": \"[0.0881, 0.9045]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.097, 0.4882]\", \"lift_point\": \"[0.097, 0.4882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0881, 0.9045]\", \"lift_point\": \"[0.0881, 0.9045]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.097, 0.4882]\", \"lift_point\": \"[0.097, 0.4882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0881, 0.9045]\", \"lift_point\": \"[0.0881, 0.9045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1928, 0.3428]\", \"lift_point\": \"[0.1928, 0.3428]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.097, 0.4882]\", \"lift_point\": \"[0.097, 0.4882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0881, 0.9045]\", \"lift_point\": \"[0.0881, 0.9045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1928, 0.3428]\", \"lift_point\": \"[0.1928, 0.3428]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4473, 0.2915]\", \"lift_point\": \"[0.4473, 0.2915]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.097, 0.4882]\", \"lift_point\": \"[0.097, 0.4882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0881, 0.9045]\", \"lift_point\": \"[0.0881, 0.9045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1928, 0.3428]\", \"lift_point\": \"[0.1928, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4473, 0.2915]\", \"lift_point\": \"[0.4473, 0.2915]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.4682]\", \"lift_point\": \"[0.8021, 0.4682]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.724, 0.6307]\", \"lift_point\": \"[0.724, 0.6307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.097, 0.4882]\", \"lift_point\": \"[0.097, 0.4882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0881, 0.9045]\", \"lift_point\": \"[0.0881, 0.9045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Seoul to Barcelona\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1928, 0.3428]\", \"lift_point\": \"[0.1928, 0.3428]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4473, 0.2915]\", \"lift_point\": \"[0.4473, 0.2915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.4682]\", \"lift_point\": \"[0.8021, 0.4682]\", \"typed_text\": \"\" Goal: Search for flights from Seoul to Barcelona>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40080_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40116_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.718, 0.6036]\", \"lift_point\": \"[0.718, 0.6036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40116_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.718, 0.6036]\", \"lift_point\": \"[0.718, 0.6036]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0949, 0.3928]\", \"lift_point\": \"[0.0949, 0.3928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40116_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.718, 0.6036]\", \"lift_point\": \"[0.718, 0.6036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0949, 0.3928]\", \"lift_point\": \"[0.0949, 0.3928]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.907]\", \"lift_point\": \"[0.0908, 0.907]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40116_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.718, 0.6036]\", \"lift_point\": \"[0.718, 0.6036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0949, 0.3928]\", \"lift_point\": \"[0.0949, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.907]\", \"lift_point\": \"[0.0908, 0.907]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from NYC to Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40116_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.718, 0.6036]\", \"lift_point\": \"[0.718, 0.6036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0949, 0.3928]\", \"lift_point\": \"[0.0949, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.907]\", \"lift_point\": \"[0.0908, 0.907]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from NYC to Paris\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.3426]\", \"lift_point\": \"[0.1898, 0.3426]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40116_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.718, 0.6036]\", \"lift_point\": \"[0.718, 0.6036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0949, 0.3928]\", \"lift_point\": \"[0.0949, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.907]\", \"lift_point\": \"[0.0908, 0.907]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from NYC to Paris\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1898, 0.3426]\", \"lift_point\": \"[0.1898, 0.3426]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40116_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.1487]\", \"lift_point\": \"[0.7264, 0.1487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7232, 0.1534]\", \"lift_point\": \"[0.7232, 0.1534]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7727, 0.8562]\", \"lift_point\": \"[0.7727, 0.8562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1093, 0.3357]\", \"lift_point\": \"[0.1093, 0.3357]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"spotify \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2039, 0.275]\", \"lift_point\": \"[0.2039, 0.275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0998, 0.9206]\", \"lift_point\": \"[0.0998, 0.9206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0998, 0.9206]\", \"lift_point\": \"[0.0998, 0.9206]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0998, 0.9206]\", \"lift_point\": \"[0.0998, 0.9206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8446, 0.4491]\", \"lift_point\": \"[0.8446, 0.4491]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5606, 0.2251]\", \"lift_point\": \"[0.5606, 0.2251]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0998, 0.9206]\", \"lift_point\": \"[0.0998, 0.9206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8446, 0.4491]\", \"lift_point\": \"[0.8446, 0.4491]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7431, 0.5095]\", \"lift_point\": \"[0.7431, 0.5095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0998, 0.9206]\", \"lift_point\": \"[0.0998, 0.9206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8446, 0.4491]\", \"lift_point\": \"[0.8446, 0.4491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7431, 0.5095]\", \"lift_point\": \"[0.7431, 0.5095]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6291, 0.5697]\", \"lift_point\": \"[0.6291, 0.5697]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0998, 0.9206]\", \"lift_point\": \"[0.0998, 0.9206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8446, 0.4491]\", \"lift_point\": \"[0.8446, 0.4491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7431, 0.5095]\", \"lift_point\": \"[0.7431, 0.5095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6291, 0.5697]\", \"lift_point\": \"[0.6291, 0.5697]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7164, 0.4917]\", \"lift_point\": \"[0.7164, 0.4917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3368, 0.4336]\", \"lift_point\": \"[0.3368, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0998, 0.9206]\", \"lift_point\": \"[0.0998, 0.9206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8446, 0.4491]\", \"lift_point\": \"[0.8446, 0.4491]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7431, 0.5095]\", \"lift_point\": \"[0.7431, 0.5095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6291, 0.5697]\", \"lift_point\": \"[0.6291, 0.5697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7164, 0.4917]\", \"lift_point\": \"[0.7164, 0.4917]\", \"typed_text\": \"\" Goal: Install the Spotify app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40161_22.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a train?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40205_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7122, 0.6291]\", \"lift_point\": \"[0.7122, 0.6291]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40205_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7122, 0.6291]\", \"lift_point\": \"[0.7122, 0.6291]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.4966]\", \"lift_point\": \"[0.0852, 0.4966]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40205_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7122, 0.6291]\", \"lift_point\": \"[0.7122, 0.6291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.4966]\", \"lift_point\": \"[0.0852, 0.4966]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0977, 0.9104]\", \"lift_point\": \"[0.0977, 0.9104]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40205_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7122, 0.6291]\", \"lift_point\": \"[0.7122, 0.6291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.4966]\", \"lift_point\": \"[0.0852, 0.4966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0977, 0.9104]\", \"lift_point\": \"[0.0977, 0.9104]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40205_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7122, 0.6291]\", \"lift_point\": \"[0.7122, 0.6291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.4966]\", \"lift_point\": \"[0.0852, 0.4966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0977, 0.9104]\", \"lift_point\": \"[0.0977, 0.9104]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1913, 0.4972]\", \"lift_point\": \"[0.1913, 0.4972]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40205_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7122, 0.6291]\", \"lift_point\": \"[0.7122, 0.6291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.4966]\", \"lift_point\": \"[0.0852, 0.4966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0977, 0.9104]\", \"lift_point\": \"[0.0977, 0.9104]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1913, 0.4972]\", \"lift_point\": \"[0.1913, 0.4972]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40205_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40219_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7162, 0.636]\", \"lift_point\": \"[0.7162, 0.636]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40219_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7162, 0.636]\", \"lift_point\": \"[0.7162, 0.636]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0862, 0.4742]\", \"lift_point\": \"[0.0862, 0.4742]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40219_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7162, 0.636]\", \"lift_point\": \"[0.7162, 0.636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0862, 0.4742]\", \"lift_point\": \"[0.0862, 0.4742]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0868, 0.8771]\", \"lift_point\": \"[0.0868, 0.8771]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40219_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7162, 0.636]\", \"lift_point\": \"[0.7162, 0.636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0862, 0.4742]\", \"lift_point\": \"[0.0862, 0.4742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0868, 0.8771]\", \"lift_point\": \"[0.0868, 0.8771]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40219_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7162, 0.636]\", \"lift_point\": \"[0.7162, 0.636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0862, 0.4742]\", \"lift_point\": \"[0.0862, 0.4742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0868, 0.8771]\", \"lift_point\": \"[0.0868, 0.8771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1903, 0.4455]\", \"lift_point\": \"[0.1903, 0.4455]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40219_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7306, 0.6166]\", \"lift_point\": \"[0.7306, 0.6166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7162, 0.636]\", \"lift_point\": \"[0.7162, 0.636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0862, 0.4742]\", \"lift_point\": \"[0.0862, 0.4742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0868, 0.8771]\", \"lift_point\": \"[0.0868, 0.8771]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1903, 0.4455]\", \"lift_point\": \"[0.1903, 0.4455]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40219_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Play Music app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40264_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Music app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40264_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest technology news?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3124]\", \"lift_point\": \"[0.1332, 0.3124]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3124]\", \"lift_point\": \"[0.1332, 0.3124]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5829]\", \"lift_point\": \"[0.0661, 0.5829]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4869, 0.6603]\", \"lift_point\": \"[0.4869, 0.6603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3124]\", \"lift_point\": \"[0.1332, 0.3124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5829]\", \"lift_point\": \"[0.0661, 0.5829]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2253, 0.6068]\", \"lift_point\": \"[0.2253, 0.6068]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4931, 0.699]\", \"lift_point\": \"[0.4931, 0.699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3124]\", \"lift_point\": \"[0.1332, 0.3124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5829]\", \"lift_point\": \"[0.0661, 0.5829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2253, 0.6068]\", \"lift_point\": \"[0.2253, 0.6068]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest technology news?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3124]\", \"lift_point\": \"[0.1332, 0.3124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5829]\", \"lift_point\": \"[0.0661, 0.5829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2253, 0.6068]\", \"lift_point\": \"[0.2253, 0.6068]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest technology news?\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.5595]\", \"lift_point\": \"[0.1351, 0.5595]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1429, 0.5551]\", \"lift_point\": \"[0.1429, 0.5551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.4917]\", \"lift_point\": \"[0.133, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1332, 0.3124]\", \"lift_point\": \"[0.1332, 0.3124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0661, 0.5829]\", \"lift_point\": \"[0.0661, 0.5829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2253, 0.6068]\", \"lift_point\": \"[0.2253, 0.6068]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest technology news?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.5595]\", \"lift_point\": \"[0.1351, 0.5595]\", \"typed_text\": \"\" Goal: What's the latest technology news?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40501_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6097, 0.4168]\", \"lift_point\": \"[0.6097, 0.4168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6097, 0.4168]\", \"lift_point\": \"[0.6097, 0.4168]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8518, 0.6107]\", \"lift_point\": \"[0.8518, 0.6107]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6097, 0.4168]\", \"lift_point\": \"[0.6097, 0.4168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8518, 0.6107]\", \"lift_point\": \"[0.8518, 0.6107]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2475]\", \"lift_point\": \"[0.8361, 0.2475]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.4722]\", \"lift_point\": \"[0.8352, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Vietnam\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6097, 0.4168]\", \"lift_point\": \"[0.6097, 0.4168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8518, 0.6107]\", \"lift_point\": \"[0.8518, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8361, 0.2475]\", \"lift_point\": \"[0.8361, 0.2475]\", \"typed_text\": \"\" Goal: What's the news in Vietnam?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40528_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40585_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8329, 0.549]\", \"lift_point\": \"[0.8329, 0.549]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40585_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8329, 0.549]\", \"lift_point\": \"[0.8329, 0.549]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40585_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8329, 0.549]\", \"lift_point\": \"[0.8329, 0.549]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40585_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8329, 0.549]\", \"lift_point\": \"[0.8329, 0.549]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2336, 0.3635]\", \"lift_point\": \"[0.2336, 0.3635]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40585_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8329, 0.549]\", \"lift_point\": \"[0.8329, 0.549]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Mexico\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2336, 0.3635]\", \"lift_point\": \"[0.2336, 0.3635]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40585_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameXplain?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8134, 0.5055]\", \"lift_point\": \"[0.8134, 0.5055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40764_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8134, 0.5055]\", \"lift_point\": \"[0.8134, 0.5055]\", \"typed_text\": \"\" Goal: What's the latest video from GameXplain?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameXplain\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40764_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8134, 0.5055]\", \"lift_point\": \"[0.8134, 0.5055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameXplain\" Goal: What's the latest video from GameXplain?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40764_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8134, 0.5055]\", \"lift_point\": \"[0.8134, 0.5055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameXplain\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from GameXplain?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4703, 0.3358]\", \"lift_point\": \"[0.4703, 0.3358]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40764_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8134, 0.5055]\", \"lift_point\": \"[0.8134, 0.5055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameXplain\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4703, 0.3358]\", \"lift_point\": \"[0.4703, 0.3358]\", \"typed_text\": \"\" Goal: What's the latest video from GameXplain?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40764_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40851_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8241, 0.5456]\", \"lift_point\": \"[0.8241, 0.5456]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40851_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8241, 0.5456]\", \"lift_point\": \"[0.8241, 0.5456]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Denver\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40851_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8241, 0.5456]\", \"lift_point\": \"[0.8241, 0.5456]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Denver\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40851_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8241, 0.5456]\", \"lift_point\": \"[0.8241, 0.5456]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Denver\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/40851_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [PRESS_HOME,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41033_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41033_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41049_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6414]\", \"lift_point\": \"[0.8171, 0.6414]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41049_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6414]\", \"lift_point\": \"[0.8171, 0.6414]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41049_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6414]\", \"lift_point\": \"[0.8171, 0.6414]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in New Jersey?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41049_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6414]\", \"lift_point\": \"[0.8171, 0.6414]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in New Jersey?\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41049_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6414]\", \"lift_point\": \"[0.8171, 0.6414]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in New Jersey?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41049_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the clock>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41105_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41105_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4751, 0.8924]\", \"lift_point\": \"[0.4751, 0.8924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41105_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4751, 0.8924]\", \"lift_point\": \"[0.4751, 0.8924]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41105_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41265_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8272, 0.558]\", \"lift_point\": \"[0.8272, 0.558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41265_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8272, 0.558]\", \"lift_point\": \"[0.8272, 0.558]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Trailers\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41265_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8272, 0.558]\", \"lift_point\": \"[0.8272, 0.558]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Trailers\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41265_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8272, 0.558]\", \"lift_point\": \"[0.8272, 0.558]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Trailers\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41265_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41275_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6055]\", \"lift_point\": \"[0.7985, 0.6055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41275_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6055]\", \"lift_point\": \"[0.7985, 0.6055]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Phoenix?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41275_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6055]\", \"lift_point\": \"[0.7985, 0.6055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Phoenix?\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41275_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7985, 0.6055]\", \"lift_point\": \"[0.7985, 0.6055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Phoenix?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41275_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8185, 0.6151]\", \"lift_point\": \"[0.8185, 0.6151]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8185, 0.6151]\", \"lift_point\": \"[0.8185, 0.6151]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8185, 0.6151]\", \"lift_point\": \"[0.8185, 0.6151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8185, 0.6151]\", \"lift_point\": \"[0.8185, 0.6151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8185, 0.6151]\", \"lift_point\": \"[0.8185, 0.6151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8185, 0.6151]\", \"lift_point\": \"[0.8185, 0.6151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7048, 0.6341]\", \"lift_point\": \"[0.7048, 0.6341]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8185, 0.6151]\", \"lift_point\": \"[0.8185, 0.6151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Home Depot\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7048, 0.6341]\", \"lift_point\": \"[0.7048, 0.6341]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41334_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the settings>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41423_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41423_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41423_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6262, 0.4764]\", \"lift_point\": \"[0.6262, 0.4764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41423_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6262, 0.4764]\", \"lift_point\": \"[0.6262, 0.4764]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41423_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0913, 0.118]\", \"lift_point\": \"[0.0913, 0.118]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0913, 0.118]\", \"lift_point\": \"[0.0913, 0.118]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0913, 0.118]\", \"lift_point\": \"[0.0913, 0.118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0913, 0.118]\", \"lift_point\": \"[0.0913, 0.118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2671, 0.8472]\", \"lift_point\": \"[0.2671, 0.8472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0913, 0.118]\", \"lift_point\": \"[0.0913, 0.118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0913, 0.118]\", \"lift_point\": \"[0.0913, 0.118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4014, 0.2675]\", \"lift_point\": \"[0.4014, 0.2675]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0913, 0.118]\", \"lift_point\": \"[0.0913, 0.118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4014, 0.2675]\", \"lift_point\": \"[0.4014, 0.2675]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41459_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41517_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.502]\", \"lift_point\": \"[0.81, 0.502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41517_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.502]\", \"lift_point\": \"[0.81, 0.502]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the LG TV\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41517_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.502]\", \"lift_point\": \"[0.81, 0.502]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the LG TV\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41517_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.502]\", \"lift_point\": \"[0.81, 0.502]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the LG TV\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the LG TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41517_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Boston to Zurich>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41586_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Boston to Zurich>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8188, 0.69]\", \"lift_point\": \"[0.8188, 0.69]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41586_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8188, 0.69]\", \"lift_point\": \"[0.8188, 0.69]\", \"typed_text\": \"\" Goal: Search for flights from Boston to Zurich>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Boston to Zurich\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41586_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8188, 0.69]\", \"lift_point\": \"[0.8188, 0.69]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Boston to Zurich\" Goal: Search for flights from Boston to Zurich>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41586_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8188, 0.69]\", \"lift_point\": \"[0.8188, 0.69]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Boston to Zurich\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Boston to Zurich>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41586_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41666_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41666_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4856, 0.685]\", \"lift_point\": \"[0.4856, 0.685]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41666_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4856, 0.685]\", \"lift_point\": \"[0.4856, 0.685]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9082]\", \"lift_point\": \"[0.1082, 0.9082]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41666_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4856, 0.685]\", \"lift_point\": \"[0.4856, 0.685]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9082]\", \"lift_point\": \"[0.1082, 0.9082]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4867]\", \"lift_point\": \"[0.4166, 0.4867]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41666_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4856, 0.685]\", \"lift_point\": \"[0.4856, 0.685]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9082]\", \"lift_point\": \"[0.1082, 0.9082]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4166, 0.4867]\", \"lift_point\": \"[0.4166, 0.4867]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41666_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8322, 0.5738]\", \"lift_point\": \"[0.8322, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Buenos aires\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7433, 0.4992]\", \"lift_point\": \"[0.7433, 0.4992]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7433, 0.4992]\", \"lift_point\": \"[0.7433, 0.4992]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41679_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41689_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8314, 0.6554]\", \"lift_point\": \"[0.8314, 0.6554]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41689_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8314, 0.6554]\", \"lift_point\": \"[0.8314, 0.6554]\", \"typed_text\": \"\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Subway\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41689_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8314, 0.6554]\", \"lift_point\": \"[0.8314, 0.6554]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Subway\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41689_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8314, 0.6554]\", \"lift_point\": \"[0.8314, 0.6554]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Subway\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Subway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41689_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated seafood restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41958_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated seafood restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.47, 0.6654]\", \"lift_point\": \"[0.47, 0.6654]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41958_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.47, 0.6654]\", \"lift_point\": \"[0.47, 0.6654]\", \"typed_text\": \"\" Goal: Search for top rated seafood restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41958_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.47, 0.6654]\", \"lift_point\": \"[0.47, 0.6654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated seafood restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.5503]\", \"lift_point\": \"[0.1816, 0.5503]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41958_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.47, 0.6654]\", \"lift_point\": \"[0.47, 0.6654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.5503]\", \"lift_point\": \"[0.1816, 0.5503]\", \"typed_text\": \"\" Goal: Search for top rated seafood restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41958_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.47, 0.6654]\", \"lift_point\": \"[0.47, 0.6654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.5503]\", \"lift_point\": \"[0.1816, 0.5503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated seafood restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3422, 0.5365]\", \"lift_point\": \"[0.3422, 0.5365]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41958_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.47, 0.6654]\", \"lift_point\": \"[0.47, 0.6654]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.5503]\", \"lift_point\": \"[0.1816, 0.5503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3422, 0.5365]\", \"lift_point\": \"[0.3422, 0.5365]\", \"typed_text\": \"\" Goal: Search for top rated seafood restaurants on Google Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/41958_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6839]\", \"lift_point\": \"[0.7832, 0.6839]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0455, 0.3601]\", \"lift_point\": \"[0.0455, 0.3601]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.4487]\", \"lift_point\": \"[0.1339, 0.4487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2592]\", \"lift_point\": \"[0.3909, 0.2592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.1584]\", \"lift_point\": \"[0.0601, 0.1584]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2991, 0.2567]\", \"lift_point\": \"[0.2991, 0.2567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1196, 0.2903]\", \"lift_point\": \"[0.1196, 0.2903]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.23, 0.2453]\", \"lift_point\": \"[0.23, 0.2453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.7465]\", \"lift_point\": \"[0.3651, 0.7465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1978, 0.2101]\", \"lift_point\": \"[0.1978, 0.2101]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3096, 0.2268]\", \"lift_point\": \"[0.3096, 0.2268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.525, 0.241]\", \"lift_point\": \"[0.525, 0.241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6438, 0.6281]\", \"lift_point\": \"[0.6438, 0.6281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.9252]\", \"lift_point\": \"[0.9011, 0.9252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7054, 0.2341]\", \"lift_point\": \"[0.7054, 0.2341]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.9252]\", \"lift_point\": \"[0.9011, 0.9252]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.898]\", \"lift_point\": \"[0.1398, 0.898]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.14, 0.8966]\", \"lift_point\": \"[0.14, 0.8966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.9252]\", \"lift_point\": \"[0.9011, 0.9252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.898]\", \"lift_point\": \"[0.1398, 0.898]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.9252]\", \"lift_point\": \"[0.9011, 0.9252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.898]\", \"lift_point\": \"[0.1398, 0.898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7361, 0.454]\", \"lift_point\": \"[0.7361, 0.454]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6212, 0.7406]\", \"lift_point\": \"[0.6212, 0.7406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6066, 0.2597]\", \"lift_point\": \"[0.6066, 0.2597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6314, 0.7128]\", \"lift_point\": \"[0.6314, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9011, 0.9252]\", \"lift_point\": \"[0.9011, 0.9252]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.898]\", \"lift_point\": \"[0.1398, 0.898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7361, 0.454]\", \"lift_point\": \"[0.7361, 0.454]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42046_32.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2746, 0.259]\", \"lift_point\": \"[0.2746, 0.259]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6902]\", \"lift_point\": \"[0.778, 0.6902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2746, 0.259]\", \"lift_point\": \"[0.2746, 0.259]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7239, 0.2662]\", \"lift_point\": \"[0.7239, 0.2662]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0411, 0.531]\", \"lift_point\": \"[0.0411, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2746, 0.259]\", \"lift_point\": \"[0.2746, 0.259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7239, 0.2662]\", \"lift_point\": \"[0.7239, 0.2662]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" flights from Buenos aires to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2746, 0.259]\", \"lift_point\": \"[0.2746, 0.259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7239, 0.2662]\", \"lift_point\": \"[0.7239, 0.2662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Helsinki\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.1962]\", \"lift_point\": \"[0.2588, 0.1962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1397, 0.3327]\", \"lift_point\": \"[0.1397, 0.3327]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2746, 0.259]\", \"lift_point\": \"[0.2746, 0.259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7239, 0.2662]\", \"lift_point\": \"[0.7239, 0.2662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.1962]\", \"lift_point\": \"[0.2588, 0.1962]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9072, 0.4807]\", \"lift_point\": \"[0.9072, 0.4807]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3953, 0.3325]\", \"lift_point\": \"[0.3953, 0.3325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2746, 0.259]\", \"lift_point\": \"[0.2746, 0.259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7239, 0.2662]\", \"lift_point\": \"[0.7239, 0.2662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.1962]\", \"lift_point\": \"[0.2588, 0.1962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9072, 0.4807]\", \"lift_point\": \"[0.9072, 0.4807]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.8324]\", \"lift_point\": \"[0.7904, 0.8324]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.643, 0.3318]\", \"lift_point\": \"[0.643, 0.3318]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2746, 0.259]\", \"lift_point\": \"[0.2746, 0.259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7239, 0.2662]\", \"lift_point\": \"[0.7239, 0.2662]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.1962]\", \"lift_point\": \"[0.2588, 0.1962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9072, 0.4807]\", \"lift_point\": \"[0.9072, 0.4807]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.8324]\", \"lift_point\": \"[0.7904, 0.8324]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42117_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3072, 0.2622]\", \"lift_point\": \"[0.3072, 0.2622]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.6808]\", \"lift_point\": \"[0.782, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3072, 0.2622]\", \"lift_point\": \"[0.3072, 0.2622]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.5296]\", \"lift_point\": \"[0.5543, 0.5296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0505, 0.4595]\", \"lift_point\": \"[0.0505, 0.4595]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3072, 0.2622]\", \"lift_point\": \"[0.3072, 0.2622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.5296]\", \"lift_point\": \"[0.5543, 0.5296]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4911, 0.5045]\", \"lift_point\": \"[0.4911, 0.5045]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"starbucks menu\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3072, 0.2622]\", \"lift_point\": \"[0.3072, 0.2622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.5296]\", \"lift_point\": \"[0.5543, 0.5296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4911, 0.5045]\", \"lift_point\": \"[0.4911, 0.5045]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1421, 0.4265]\", \"lift_point\": \"[0.1421, 0.4265]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3072, 0.2622]\", \"lift_point\": \"[0.3072, 0.2622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.5296]\", \"lift_point\": \"[0.5543, 0.5296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4911, 0.5045]\", \"lift_point\": \"[0.4911, 0.5045]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3072, 0.2622]\", \"lift_point\": \"[0.3072, 0.2622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.5296]\", \"lift_point\": \"[0.5543, 0.5296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4911, 0.5045]\", \"lift_point\": \"[0.4911, 0.5045]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2505, 0.1282]\", \"lift_point\": \"[0.2505, 0.1282]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3342, 0.2367]\", \"lift_point\": \"[0.3342, 0.2367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3072, 0.2622]\", \"lift_point\": \"[0.3072, 0.2622]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.5296]\", \"lift_point\": \"[0.5543, 0.5296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4911, 0.5045]\", \"lift_point\": \"[0.4911, 0.5045]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2505, 0.1282]\", \"lift_point\": \"[0.2505, 0.1282]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42353_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42436_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.3838]\", \"lift_point\": \"[0.6475, 0.3838]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42436_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.3838]\", \"lift_point\": \"[0.6475, 0.3838]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0486, 0.9125]\", \"lift_point\": \"[0.0486, 0.9125]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42436_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.3838]\", \"lift_point\": \"[0.6475, 0.3838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0486, 0.9125]\", \"lift_point\": \"[0.0486, 0.9125]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.3663]\", \"lift_point\": \"[0.0525, 0.3663]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42436_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.3838]\", \"lift_point\": \"[0.6475, 0.3838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0486, 0.9125]\", \"lift_point\": \"[0.0486, 0.9125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.3663]\", \"lift_point\": \"[0.0525, 0.3663]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best burger restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42436_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.3838]\", \"lift_point\": \"[0.6475, 0.3838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0486, 0.9125]\", \"lift_point\": \"[0.0486, 0.9125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.3663]\", \"lift_point\": \"[0.0525, 0.3663]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best burger restaurants\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.3866]\", \"lift_point\": \"[0.1121, 0.3866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42436_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6475, 0.3838]\", \"lift_point\": \"[0.6475, 0.3838]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0486, 0.9125]\", \"lift_point\": \"[0.0486, 0.9125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.3663]\", \"lift_point\": \"[0.0525, 0.3663]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best burger restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1121, 0.3866]\", \"lift_point\": \"[0.1121, 0.3866]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42436_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42441_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9357, 0.4492]\", \"lift_point\": \"[0.9357, 0.4492]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42441_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9357, 0.4492]\", \"lift_point\": \"[0.9357, 0.4492]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42441_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9357, 0.4492]\", \"lift_point\": \"[0.9357, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.3204]\", \"lift_point\": \"[0.3575, 0.3204]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42441_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9357, 0.4492]\", \"lift_point\": \"[0.9357, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.3204]\", \"lift_point\": \"[0.3575, 0.3204]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42441_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5016, 0.487]\", \"lift_point\": \"[0.5016, 0.487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0826, 0.1155]\", \"lift_point\": \"[0.0826, 0.1155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2721, 0.5962]\", \"lift_point\": \"[0.2721, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0809, 0.8737]\", \"lift_point\": \"[0.0809, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0795, 0.8737]\", \"lift_point\": \"[0.0795, 0.8737]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.078, 0.8768]\", \"lift_point\": \"[0.078, 0.8768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0833, 0.8883]\", \"lift_point\": \"[0.0833, 0.8883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0818, 0.8851]\", \"lift_point\": \"[0.0818, 0.8851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.07, 0.9181]\", \"lift_point\": \"[0.07, 0.9181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.9055]\", \"lift_point\": \"[0.0715, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_54.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0766, 0.873]\", \"lift_point\": \"[0.0766, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_55.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_56.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3219, 0.6493]\", \"lift_point\": \"[0.3219, 0.6493]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_57.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3219, 0.6493]\", \"lift_point\": \"[0.3219, 0.6493]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2395, 0.117]\", \"lift_point\": \"[0.2395, 0.117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_58.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3219, 0.6493]\", \"lift_point\": \"[0.3219, 0.6493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2395, 0.117]\", \"lift_point\": \"[0.2395, 0.117]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_59.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3219, 0.6493]\", \"lift_point\": \"[0.3219, 0.6493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2395, 0.117]\", \"lift_point\": \"[0.2395, 0.117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.07]\", \"lift_point\": \"[0.6477, 0.07]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_60.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3219, 0.6493]\", \"lift_point\": \"[0.3219, 0.6493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2395, 0.117]\", \"lift_point\": \"[0.2395, 0.117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.07]\", \"lift_point\": \"[0.6477, 0.07]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.662, 0.09]\", \"lift_point\": \"[0.662, 0.09]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_61.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.798, 0.4661]\", \"lift_point\": \"[0.798, 0.4661]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3219, 0.6493]\", \"lift_point\": \"[0.3219, 0.6493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2395, 0.117]\", \"lift_point\": \"[0.2395, 0.117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.07]\", \"lift_point\": \"[0.6477, 0.07]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.662, 0.09]\", \"lift_point\": \"[0.662, 0.09]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6593, 0.121]\", \"lift_point\": \"[0.6593, 0.121]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_62.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.076, 0.0763]\", \"lift_point\": \"[0.076, 0.0763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3219, 0.6493]\", \"lift_point\": \"[0.3219, 0.6493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2395, 0.117]\", \"lift_point\": \"[0.2395, 0.117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.07]\", \"lift_point\": \"[0.6477, 0.07]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.662, 0.09]\", \"lift_point\": \"[0.662, 0.09]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6593, 0.121]\", \"lift_point\": \"[0.6593, 0.121]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42617_63.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.358, 0.2776]\", \"lift_point\": \"[0.358, 0.2776]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8613, 0.5473]\", \"lift_point\": \"[0.8613, 0.5473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.358, 0.2776]\", \"lift_point\": \"[0.358, 0.2776]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new eyeshadow on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.358, 0.2776]\", \"lift_point\": \"[0.358, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6703, 0.5536]\", \"lift_point\": \"[0.6703, 0.5536]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.358, 0.2776]\", \"lift_point\": \"[0.358, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6703, 0.5536]\", \"lift_point\": \"[0.6703, 0.5536]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.358, 0.2776]\", \"lift_point\": \"[0.358, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6703, 0.5536]\", \"lift_point\": \"[0.6703, 0.5536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5583, 0.5861]\", \"lift_point\": \"[0.5583, 0.5861]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.4962]\", \"lift_point\": \"[0.4399, 0.4962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.358, 0.2776]\", \"lift_point\": \"[0.358, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6703, 0.5536]\", \"lift_point\": \"[0.6703, 0.5536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5583, 0.5861]\", \"lift_point\": \"[0.5583, 0.5861]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42696_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8706, 0.5317]\", \"lift_point\": \"[0.8706, 0.5317]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of China\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1411, 0.397]\", \"lift_point\": \"[0.1411, 0.397]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6228, 0.485]\", \"lift_point\": \"[0.6228, 0.485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6551]\", \"lift_point\": \"[0.7647, 0.6551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4288, 0.9313]\", \"lift_point\": \"[0.4288, 0.9313]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4303, 0.9281]\", \"lift_point\": \"[0.4303, 0.9281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.429, 0.922]\", \"lift_point\": \"[0.429, 0.922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4275, 0.9255]\", \"lift_point\": \"[0.4275, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_54.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_55.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_56.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_57.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_58.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_59.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_60.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_61.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_62.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_63.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9286]\", \"lift_point\": \"[0.4936, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.9317]\", \"lift_point\": \"[0.4936, 0.9317]\", \"typed_text\": \"\" Goal: What is the capital of China?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42767_64.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.4239]\", \"lift_point\": \"[0.7749, 0.4239]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42885_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.4239]\", \"lift_point\": \"[0.7749, 0.4239]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42885_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.4239]\", \"lift_point\": \"[0.7749, 0.4239]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42885_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.4239]\", \"lift_point\": \"[0.7749, 0.4239]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3149, 0.5286]\", \"lift_point\": \"[0.3149, 0.5286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42885_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.4239]\", \"lift_point\": \"[0.7749, 0.4239]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Buenos aires to Helsinki\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3149, 0.5286]\", \"lift_point\": \"[0.3149, 0.5286]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42885_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2725, 0.465]\", \"lift_point\": \"[0.2725, 0.465]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2725, 0.465]\", \"lift_point\": \"[0.2725, 0.465]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3037, 0.7113]\", \"lift_point\": \"[0.3037, 0.7113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2725, 0.465]\", \"lift_point\": \"[0.2725, 0.465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3037, 0.7113]\", \"lift_point\": \"[0.3037, 0.7113]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.7297]\", \"lift_point\": \"[0.8191, 0.7297]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2725, 0.465]\", \"lift_point\": \"[0.2725, 0.465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3037, 0.7113]\", \"lift_point\": \"[0.3037, 0.7113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.7297]\", \"lift_point\": \"[0.8191, 0.7297]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7044, 0.5915]\", \"lift_point\": \"[0.7044, 0.5915]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8296, 0.8163]\", \"lift_point\": \"[0.8296, 0.8163]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.6874]\", \"lift_point\": \"[0.1233, 0.6874]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Lyft app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2725, 0.465]\", \"lift_point\": \"[0.2725, 0.465]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3037, 0.7113]\", \"lift_point\": \"[0.3037, 0.7113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8191, 0.7297]\", \"lift_point\": \"[0.8191, 0.7297]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7044, 0.5915]\", \"lift_point\": \"[0.7044, 0.5915]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/42916_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.4833]\", \"lift_point\": \"[0.7904, 0.4833]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43001_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.4833]\", \"lift_point\": \"[0.7904, 0.4833]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43001_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.4833]\", \"lift_point\": \"[0.7904, 0.4833]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Brazil\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43001_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.4833]\", \"lift_point\": \"[0.7904, 0.4833]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Brazil\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43001_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.4833]\", \"lift_point\": \"[0.7904, 0.4833]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Brazil\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.284, 0.4831]\", \"lift_point\": \"[0.284, 0.4831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43001_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7904, 0.4833]\", \"lift_point\": \"[0.7904, 0.4833]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Brazil\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.284, 0.4831]\", \"lift_point\": \"[0.284, 0.4831]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43001_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1726, 0.6238]\", \"lift_point\": \"[0.1726, 0.6238]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1726, 0.6238]\", \"lift_point\": \"[0.1726, 0.6238]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9377]\", \"lift_point\": \"[0.1082, 0.9377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1726, 0.6238]\", \"lift_point\": \"[0.1726, 0.6238]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9377]\", \"lift_point\": \"[0.1082, 0.9377]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a cheetah?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1726, 0.6238]\", \"lift_point\": \"[0.1726, 0.6238]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9377]\", \"lift_point\": \"[0.1082, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a cheetah?\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1726, 0.6238]\", \"lift_point\": \"[0.1726, 0.6238]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9377]\", \"lift_point\": \"[0.1082, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a cheetah?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2151, 0.4211]\", \"lift_point\": \"[0.2151, 0.4211]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3455, 0.4821]\", \"lift_point\": \"[0.3455, 0.4821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1726, 0.6238]\", \"lift_point\": \"[0.1726, 0.6238]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9377]\", \"lift_point\": \"[0.1082, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a cheetah?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2151, 0.4211]\", \"lift_point\": \"[0.2151, 0.4211]\", \"typed_text\": \"\" Goal: What is the speed of a cheetah?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43177_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.7153]\", \"lift_point\": \"[0.7988, 0.7153]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43229_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.7153]\", \"lift_point\": \"[0.7988, 0.7153]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43229_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.7153]\", \"lift_point\": \"[0.7988, 0.7153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43229_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.7153]\", \"lift_point\": \"[0.7988, 0.7153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43229_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.7153]\", \"lift_point\": \"[0.7988, 0.7153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.6393]\", \"lift_point\": \"[0.1436, 0.6393]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43229_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.7153]\", \"lift_point\": \"[0.7988, 0.7153]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.5275]\", \"lift_point\": \"[0.0567, 0.5275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.6393]\", \"lift_point\": \"[0.1436, 0.6393]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43229_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new mascara>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43680_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6922]\", \"lift_point\": \"[0.7825, 0.6922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43680_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6922]\", \"lift_point\": \"[0.7825, 0.6922]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7673, 0.7003]\", \"lift_point\": \"[0.7673, 0.7003]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43680_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6922]\", \"lift_point\": \"[0.7825, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7673, 0.7003]\", \"lift_point\": \"[0.7673, 0.7003]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.5568]\", \"lift_point\": \"[0.065, 0.5568]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43680_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6922]\", \"lift_point\": \"[0.7825, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7673, 0.7003]\", \"lift_point\": \"[0.7673, 0.7003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.5568]\", \"lift_point\": \"[0.065, 0.5568]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43680_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6922]\", \"lift_point\": \"[0.7825, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7673, 0.7003]\", \"lift_point\": \"[0.7673, 0.7003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.5568]\", \"lift_point\": \"[0.065, 0.5568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43680_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6922]\", \"lift_point\": \"[0.7825, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7673, 0.7003]\", \"lift_point\": \"[0.7673, 0.7003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.5568]\", \"lift_point\": \"[0.065, 0.5568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43680_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43778_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43778_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3726, 0.4941]\", \"lift_point\": \"[0.3726, 0.4941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43778_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3726, 0.4941]\", \"lift_point\": \"[0.3726, 0.4941]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4982]\", \"lift_point\": \"[0.0721, 0.4982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43778_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3726, 0.4941]\", \"lift_point\": \"[0.3726, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4982]\", \"lift_point\": \"[0.0721, 0.4982]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43778_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3726, 0.4941]\", \"lift_point\": \"[0.3726, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4982]\", \"lift_point\": \"[0.0721, 0.4982]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43778_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3726, 0.4941]\", \"lift_point\": \"[0.3726, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4982]\", \"lift_point\": \"[0.0721, 0.4982]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43778_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.3924]\", \"lift_point\": \"[0.0651, 0.3924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43826_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.3924]\", \"lift_point\": \"[0.0651, 0.3924]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43826_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.3924]\", \"lift_point\": \"[0.0651, 0.3924]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43826_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.3924]\", \"lift_point\": \"[0.0651, 0.3924]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Diego to Seattle\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/43826_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4996]\", \"lift_point\": \"[0.3395, 0.4996]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4996]\", \"lift_point\": \"[0.3395, 0.4996]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3527, 0.5164]\", \"lift_point\": \"[0.3527, 0.5164]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4996]\", \"lift_point\": \"[0.3395, 0.4996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3527, 0.5164]\", \"lift_point\": \"[0.3527, 0.5164]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4573]\", \"lift_point\": \"[0.0646, 0.4573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4996]\", \"lift_point\": \"[0.3395, 0.4996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3527, 0.5164]\", \"lift_point\": \"[0.3527, 0.5164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4573]\", \"lift_point\": \"[0.0646, 0.4573]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4996]\", \"lift_point\": \"[0.3395, 0.4996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3527, 0.5164]\", \"lift_point\": \"[0.3527, 0.5164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4573]\", \"lift_point\": \"[0.0646, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4996]\", \"lift_point\": \"[0.3395, 0.4996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3527, 0.5164]\", \"lift_point\": \"[0.3527, 0.5164]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4573]\", \"lift_point\": \"[0.0646, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44048_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44139_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3541, 0.4888]\", \"lift_point\": \"[0.3541, 0.4888]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44139_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3541, 0.4888]\", \"lift_point\": \"[0.3541, 0.4888]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5427]\", \"lift_point\": \"[0.0589, 0.5427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44139_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3541, 0.4888]\", \"lift_point\": \"[0.3541, 0.4888]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5427]\", \"lift_point\": \"[0.0589, 0.5427]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurnat in Dallas?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44139_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3541, 0.4888]\", \"lift_point\": \"[0.3541, 0.4888]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5427]\", \"lift_point\": \"[0.0589, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurnat in Dallas?\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44139_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3541, 0.4888]\", \"lift_point\": \"[0.3541, 0.4888]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.5427]\", \"lift_point\": \"[0.0589, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurnat in Dallas?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Dallas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44139_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Peru>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of peru\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44364_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of peru\" Goal: Google the capital of Peru>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.3309]\", \"lift_point\": \"[0.1424, 0.3309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44364_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of peru\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.3309]\", \"lift_point\": \"[0.1424, 0.3309]\", \"typed_text\": \"\" Goal: Google the capital of Peru>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44364_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2098, 0.2994]\", \"lift_point\": \"[0.2098, 0.2994]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2098, 0.2994]\", \"lift_point\": \"[0.2098, 0.2994]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3194, 0.3224]\", \"lift_point\": \"[0.3194, 0.3224]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2098, 0.2994]\", \"lift_point\": \"[0.2098, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3194, 0.3224]\", \"lift_point\": \"[0.3194, 0.3224]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2419, 0.828]\", \"lift_point\": \"[0.2419, 0.828]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8857, 0.2075]\", \"lift_point\": \"[0.8857, 0.2075]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2098, 0.2994]\", \"lift_point\": \"[0.2098, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3194, 0.3224]\", \"lift_point\": \"[0.3194, 0.3224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2419, 0.828]\", \"lift_point\": \"[0.2419, 0.828]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2412, 0.8302]\", \"lift_point\": \"[0.2412, 0.8302]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8786, 0.138]\", \"lift_point\": \"[0.8786, 0.138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2098, 0.2994]\", \"lift_point\": \"[0.2098, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3194, 0.3224]\", \"lift_point\": \"[0.3194, 0.3224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2419, 0.828]\", \"lift_point\": \"[0.2419, 0.828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2412, 0.8302]\", \"lift_point\": \"[0.2412, 0.8302]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5936, 0.9015]\", \"lift_point\": \"[0.5936, 0.9015]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5964]\", \"lift_point\": \"[0.8927, 0.5964]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2098, 0.2994]\", \"lift_point\": \"[0.2098, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3194, 0.3224]\", \"lift_point\": \"[0.3194, 0.3224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2419, 0.828]\", \"lift_point\": \"[0.2419, 0.828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2412, 0.8302]\", \"lift_point\": \"[0.2412, 0.8302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5936, 0.9015]\", \"lift_point\": \"[0.5936, 0.9015]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1578, 0.9112]\", \"lift_point\": \"[0.1578, 0.9112]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"t for in boston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2098, 0.2994]\", \"lift_point\": \"[0.2098, 0.2994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3194, 0.3224]\", \"lift_point\": \"[0.3194, 0.3224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2419, 0.828]\", \"lift_point\": \"[0.2419, 0.828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2412, 0.8302]\", \"lift_point\": \"[0.2412, 0.8302]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5936, 0.9015]\", \"lift_point\": \"[0.5936, 0.9015]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1578, 0.9112]\", \"lift_point\": \"[0.1578, 0.9112]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44392_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44565_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44565_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3761, 0.5204]\", \"lift_point\": \"[0.3761, 0.5204]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44565_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3761, 0.5204]\", \"lift_point\": \"[0.3761, 0.5204]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9235]\", \"lift_point\": \"[0.0708, 0.9235]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44565_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3761, 0.5204]\", \"lift_point\": \"[0.3761, 0.5204]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9235]\", \"lift_point\": \"[0.0708, 0.9235]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.4264]\", \"lift_point\": \"[0.134, 0.4264]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44565_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3761, 0.5204]\", \"lift_point\": \"[0.3761, 0.5204]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9235]\", \"lift_point\": \"[0.0708, 0.9235]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.4264]\", \"lift_point\": \"[0.134, 0.4264]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44565_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Facebook app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6519, 0.8804]\", \"lift_point\": \"[0.6519, 0.8804]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6519, 0.8804]\", \"lift_point\": \"[0.6519, 0.8804]\", \"typed_text\": \"\" Goal: Install the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.5344]\", \"lift_point\": \"[0.0703, 0.5344]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6519, 0.8804]\", \"lift_point\": \"[0.6519, 0.8804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.5344]\", \"lift_point\": \"[0.0703, 0.5344]\", \"typed_text\": \"\" Goal: Install the Facebook app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6519, 0.8804]\", \"lift_point\": \"[0.6519, 0.8804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.5344]\", \"lift_point\": \"[0.0703, 0.5344]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook app\" Goal: Install the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1527, 0.3694]\", \"lift_point\": \"[0.1527, 0.3694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6519, 0.8804]\", \"lift_point\": \"[0.6519, 0.8804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.5344]\", \"lift_point\": \"[0.0703, 0.5344]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1527, 0.3694]\", \"lift_point\": \"[0.1527, 0.3694]\", \"typed_text\": \"\" Goal: Install the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1676, 0.852]\", \"lift_point\": \"[0.1676, 0.852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6519, 0.8804]\", \"lift_point\": \"[0.6519, 0.8804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.5344]\", \"lift_point\": \"[0.0703, 0.5344]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1527, 0.3694]\", \"lift_point\": \"[0.1527, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1676, 0.852]\", \"lift_point\": \"[0.1676, 0.852]\", \"typed_text\": \"\" Goal: Install the Facebook app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44611_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.5926]\", \"lift_point\": \"[0.8849, 0.5926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.5926]\", \"lift_point\": \"[0.8849, 0.5926]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.5926]\", \"lift_point\": \"[0.8849, 0.5926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.5926]\", \"lift_point\": \"[0.8849, 0.5926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.5926]\", \"lift_point\": \"[0.8849, 0.5926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6859, 0.4387]\", \"lift_point\": \"[0.6859, 0.4387]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.5926]\", \"lift_point\": \"[0.8849, 0.5926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6859, 0.4387]\", \"lift_point\": \"[0.6859, 0.4387]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.5783]\", \"lift_point\": \"[0.8356, 0.5783]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8849, 0.5926]\", \"lift_point\": \"[0.8849, 0.5926]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Zurich\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6859, 0.4387]\", \"lift_point\": \"[0.6859, 0.4387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8356, 0.5783]\", \"lift_point\": \"[0.8356, 0.5783]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44698_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44758_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4639]\", \"lift_point\": \"[0.8947, 0.4639]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44758_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4639]\", \"lift_point\": \"[0.8947, 0.4639]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of the United Kingdom\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44758_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4639]\", \"lift_point\": \"[0.8947, 0.4639]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of the United Kingdom\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44758_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4639]\", \"lift_point\": \"[0.8947, 0.4639]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of the United Kingdom\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of the United Kingdom?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44758_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Puerto Rico\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Puerto Rico\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Puerto Rico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Puerto Rico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2512, 0.6497]\", \"lift_point\": \"[0.2512, 0.6497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Puerto Rico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2512, 0.6497]\", \"lift_point\": \"[0.2512, 0.6497]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8978, 0.6783]\", \"lift_point\": \"[0.8978, 0.6783]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.2415]\", \"lift_point\": \"[0.8872, 0.2415]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Puerto Rico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2512, 0.6497]\", \"lift_point\": \"[0.2512, 0.6497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8978, 0.6783]\", \"lift_point\": \"[0.8978, 0.6783]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44772_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this weekend?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.6785]\", \"lift_point\": \"[0.3583, 0.6785]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.6785]\", \"lift_point\": \"[0.3583, 0.6785]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.6785]\", \"lift_point\": \"[0.3583, 0.6785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8129, 0.5797]\", \"lift_point\": \"[0.8129, 0.5797]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.458]\", \"lift_point\": \"[0.8785, 0.458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news this weekend\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.6785]\", \"lift_point\": \"[0.3583, 0.6785]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8129, 0.5797]\", \"lift_point\": \"[0.8129, 0.5797]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44806_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44823_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8754, 0.4069]\", \"lift_point\": \"[0.8754, 0.4069]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44823_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8754, 0.4069]\", \"lift_point\": \"[0.8754, 0.4069]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44823_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8754, 0.4069]\", \"lift_point\": \"[0.8754, 0.4069]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44823_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8754, 0.4069]\", \"lift_point\": \"[0.8754, 0.4069]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3497, 0.6944]\", \"lift_point\": \"[0.3497, 0.6944]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44823_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8754, 0.4069]\", \"lift_point\": \"[0.8754, 0.4069]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3497, 0.6944]\", \"lift_point\": \"[0.3497, 0.6944]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44823_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44874_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44874_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9141, 0.4266]\", \"lift_point\": \"[0.9141, 0.4266]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44874_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9141, 0.4266]\", \"lift_point\": \"[0.9141, 0.4266]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.4925]\", \"lift_point\": \"[0.8934, 0.4925]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44874_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9141, 0.4266]\", \"lift_point\": \"[0.9141, 0.4266]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.4925]\", \"lift_point\": \"[0.8934, 0.4925]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44874_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9141, 0.4266]\", \"lift_point\": \"[0.9141, 0.4266]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.4925]\", \"lift_point\": \"[0.8934, 0.4925]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada?\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44874_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9141, 0.4266]\", \"lift_point\": \"[0.9141, 0.4266]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.4925]\", \"lift_point\": \"[0.8934, 0.4925]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Canada?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Canada?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44874_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7326, 0.5816]\", \"lift_point\": \"[0.7326, 0.5816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8824, 0.6338]\", \"lift_point\": \"[0.8824, 0.6338]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the new iPhone on eBay\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7326, 0.5816]\", \"lift_point\": \"[0.7326, 0.5816]\", \"typed_text\": \"\" Goal: What's the price of the new iPhone on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44959_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7857, 0.6857]\", \"lift_point\": \"[0.7857, 0.6857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.5086]\", \"lift_point\": \"[0.9056, 0.5086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8936, 0.8628]\", \"lift_point\": \"[0.8936, 0.8628]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.4856]\", \"lift_point\": \"[0.2862, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2875, 0.4116]\", \"lift_point\": \"[0.2875, 0.4116]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.168, 0.3305]\", \"lift_point\": \"[0.168, 0.3305]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3686, 0.184]\", \"lift_point\": \"[0.3686, 0.184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.168, 0.3305]\", \"lift_point\": \"[0.168, 0.3305]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1798, 0.3222]\", \"lift_point\": \"[0.1798, 0.3222]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6881, 0.58]\", \"lift_point\": \"[0.6881, 0.58]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.168, 0.3305]\", \"lift_point\": \"[0.168, 0.3305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1798, 0.3222]\", \"lift_point\": \"[0.1798, 0.3222]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.168, 0.3305]\", \"lift_point\": \"[0.168, 0.3305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1798, 0.3222]\", \"lift_point\": \"[0.1798, 0.3222]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5503, 0.5891]\", \"lift_point\": \"[0.5503, 0.5891]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.168, 0.3305]\", \"lift_point\": \"[0.168, 0.3305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1798, 0.3222]\", \"lift_point\": \"[0.1798, 0.3222]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/44996_22.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Peru?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7021]\", \"lift_point\": \"[0.7989, 0.7021]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45114_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7021]\", \"lift_point\": \"[0.7989, 0.7021]\", \"typed_text\": \"\" Goal: What's the news in Peru?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0488, 0.5993]\", \"lift_point\": \"[0.0488, 0.5993]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45114_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7021]\", \"lift_point\": \"[0.7989, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0488, 0.5993]\", \"lift_point\": \"[0.0488, 0.5993]\", \"typed_text\": \"\" Goal: What's the news in Peru?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Peru\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45114_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7021]\", \"lift_point\": \"[0.7989, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0488, 0.5993]\", \"lift_point\": \"[0.0488, 0.5993]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Peru\" Goal: What's the news in Peru?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45114_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7021]\", \"lift_point\": \"[0.7989, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0488, 0.5993]\", \"lift_point\": \"[0.0488, 0.5993]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Peru\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Peru?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2382, 0.1712]\", \"lift_point\": \"[0.2382, 0.1712]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45114_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.7021]\", \"lift_point\": \"[0.7989, 0.7021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0488, 0.5993]\", \"lift_point\": \"[0.0488, 0.5993]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Peru\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2382, 0.1712]\", \"lift_point\": \"[0.2382, 0.1712]\", \"typed_text\": \"\" Goal: What's the news in Peru?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45114_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calendar>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45125_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the calendar>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45125_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.1131]\", \"lift_point\": \"[0.3583, 0.1131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45125_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.1131]\", \"lift_point\": \"[0.3583, 0.1131]\", \"typed_text\": \"\" Goal: Open the calendar>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8967, 0.926]\", \"lift_point\": \"[0.8967, 0.926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45125_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.1131]\", \"lift_point\": \"[0.3583, 0.1131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8967, 0.926]\", \"lift_point\": \"[0.8967, 0.926]\", \"typed_text\": \"\" Goal: Open the calendar>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9037, 0.9246]\", \"lift_point\": \"[0.9037, 0.9246]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45125_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.1131]\", \"lift_point\": \"[0.3583, 0.1131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8967, 0.926]\", \"lift_point\": \"[0.8967, 0.926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9037, 0.9246]\", \"lift_point\": \"[0.9037, 0.9246]\", \"typed_text\": \"\" Goal: Open the calendar>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8817, 0.5028]\", \"lift_point\": \"[0.8817, 0.5028]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45125_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3583, 0.1131]\", \"lift_point\": \"[0.3583, 0.1131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8967, 0.926]\", \"lift_point\": \"[0.8967, 0.926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9037, 0.9246]\", \"lift_point\": \"[0.9037, 0.9246]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8817, 0.5028]\", \"lift_point\": \"[0.8817, 0.5028]\", \"typed_text\": \"\" Goal: Open the calendar>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45125_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.6038]\", \"lift_point\": \"[0.0531, 0.6038]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.6038]\", \"lift_point\": \"[0.0531, 0.6038]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.921]\", \"lift_point\": \"[0.0579, 0.921]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.6038]\", \"lift_point\": \"[0.0531, 0.6038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.921]\", \"lift_point\": \"[0.0579, 0.921]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Canada\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.6038]\", \"lift_point\": \"[0.0531, 0.6038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.921]\", \"lift_point\": \"[0.0579, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Canada\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7958, 0.6923]\", \"lift_point\": \"[0.7958, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.6038]\", \"lift_point\": \"[0.0531, 0.6038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.921]\", \"lift_point\": \"[0.0579, 0.921]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Canada\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45292_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9192]\", \"lift_point\": \"[0.0598, 0.9192]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9192]\", \"lift_point\": \"[0.0598, 0.9192]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Mexico\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9192]\", \"lift_point\": \"[0.0598, 0.9192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Mexico\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9192]\", \"lift_point\": \"[0.0598, 0.9192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Mexico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6831]\", \"lift_point\": \"[0.7768, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9192]\", \"lift_point\": \"[0.0598, 0.9192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Mexico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.4897]\", \"lift_point\": \"[0.4472, 0.4897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6694]\", \"lift_point\": \"[0.7938, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9192]\", \"lift_point\": \"[0.0598, 0.9192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Mexico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.4897]\", \"lift_point\": \"[0.4472, 0.4897]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8551, 0.4548]\", \"lift_point\": \"[0.8551, 0.4548]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7989, 0.6694]\", \"lift_point\": \"[0.7989, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.5681]\", \"lift_point\": \"[0.0568, 0.5681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9192]\", \"lift_point\": \"[0.0598, 0.9192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Mexico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.4897]\", \"lift_point\": \"[0.4472, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8551, 0.4548]\", \"lift_point\": \"[0.8551, 0.4548]\", \"typed_text\": \"\" Goal: What is the capital of Mexico?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45389_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Ecuador>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7746, 0.6711]\", \"lift_point\": \"[0.7746, 0.6711]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45599_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7746, 0.6711]\", \"lift_point\": \"[0.7746, 0.6711]\", \"typed_text\": \"\" Goal: Google the capital of Ecuador>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4653]\", \"lift_point\": \"[0.0652, 0.4653]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45599_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7746, 0.6711]\", \"lift_point\": \"[0.7746, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4653]\", \"lift_point\": \"[0.0652, 0.4653]\", \"typed_text\": \"\" Goal: Google the capital of Ecuador>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Ecuador\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45599_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7746, 0.6711]\", \"lift_point\": \"[0.7746, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4653]\", \"lift_point\": \"[0.0652, 0.4653]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Ecuador\" Goal: Google the capital of Ecuador>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4096]\", \"lift_point\": \"[0.0596, 0.4096]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45599_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7746, 0.6711]\", \"lift_point\": \"[0.7746, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4653]\", \"lift_point\": \"[0.0652, 0.4653]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Ecuador\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4096]\", \"lift_point\": \"[0.0596, 0.4096]\", \"typed_text\": \"\" Goal: Google the capital of Ecuador>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1541, 0.4309]\", \"lift_point\": \"[0.1541, 0.4309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45599_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7746, 0.6711]\", \"lift_point\": \"[0.7746, 0.6711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0652, 0.4653]\", \"lift_point\": \"[0.0652, 0.4653]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Ecuador\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4096]\", \"lift_point\": \"[0.0596, 0.4096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1541, 0.4309]\", \"lift_point\": \"[0.1541, 0.4309]\", \"typed_text\": \"\" Goal: Google the capital of Ecuador>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45599_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Argentina?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45637_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8032, 0.6956]\", \"lift_point\": \"[0.8032, 0.6956]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45637_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8032, 0.6956]\", \"lift_point\": \"[0.8032, 0.6956]\", \"typed_text\": \"\" Goal: What is the capital of Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.5482]\", \"lift_point\": \"[0.0471, 0.5482]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45637_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8032, 0.6956]\", \"lift_point\": \"[0.8032, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.5482]\", \"lift_point\": \"[0.0471, 0.5482]\", \"typed_text\": \"\" Goal: What is the capital of Argentina?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Argentina ?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45637_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8032, 0.6956]\", \"lift_point\": \"[0.8032, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.5482]\", \"lift_point\": \"[0.0471, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Argentina?\" Goal: What is the capital of Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.5348]\", \"lift_point\": \"[0.1386, 0.5348]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45637_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8032, 0.6956]\", \"lift_point\": \"[0.8032, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0471, 0.5482]\", \"lift_point\": \"[0.0471, 0.5482]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Argentina?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.5348]\", \"lift_point\": \"[0.1386, 0.5348]\", \"typed_text\": \"\" Goal: What is the capital of Argentina?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45637_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45808_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.5269]\", \"lift_point\": \"[0.9271, 0.5269]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45808_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.5269]\", \"lift_point\": \"[0.9271, 0.5269]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45808_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.5269]\", \"lift_point\": \"[0.9271, 0.5269]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes?\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.115, 0.4563]\", \"lift_point\": \"[0.115, 0.4563]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45808_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9271, 0.5269]\", \"lift_point\": \"[0.9271, 0.5269]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.115, 0.4563]\", \"lift_point\": \"[0.115, 0.4563]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45808_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9006, 0.4981]\", \"lift_point\": \"[0.9006, 0.4981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9006, 0.4981]\", \"lift_point\": \"[0.9006, 0.4981]\", \"typed_text\": \"\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9088, 0.9116]\", \"lift_point\": \"[0.9088, 0.9116]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9006, 0.4981]\", \"lift_point\": \"[0.9006, 0.4981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9088, 0.9116]\", \"lift_point\": \"[0.9088, 0.9116]\", \"typed_text\": \"\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.249, 0.425]\", \"lift_point\": \"[0.249, 0.425]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9006, 0.4981]\", \"lift_point\": \"[0.9006, 0.4981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9088, 0.9116]\", \"lift_point\": \"[0.9088, 0.9116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.249, 0.425]\", \"lift_point\": \"[0.249, 0.425]\", \"typed_text\": \"\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the average speed of a car?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9006, 0.4981]\", \"lift_point\": \"[0.9006, 0.4981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9088, 0.9116]\", \"lift_point\": \"[0.9088, 0.9116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.249, 0.425]\", \"lift_point\": \"[0.249, 0.425]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the average speed of a car?\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0445, 0.7409]\", \"lift_point\": \"[0.0445, 0.7409]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9006, 0.4981]\", \"lift_point\": \"[0.9006, 0.4981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9088, 0.9116]\", \"lift_point\": \"[0.9088, 0.9116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.249, 0.425]\", \"lift_point\": \"[0.249, 0.425]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the average speed of a car?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0445, 0.7409]\", \"lift_point\": \"[0.0445, 0.7409]\", \"typed_text\": \"\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.6089]\", \"lift_point\": \"[0.108, 0.6089]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.578]\", \"lift_point\": \"[0.7994, 0.578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9006, 0.4981]\", \"lift_point\": \"[0.9006, 0.4981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9088, 0.9116]\", \"lift_point\": \"[0.9088, 0.9116]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.249, 0.425]\", \"lift_point\": \"[0.249, 0.425]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the average speed of a car?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0445, 0.7409]\", \"lift_point\": \"[0.0445, 0.7409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.6089]\", \"lift_point\": \"[0.108, 0.6089]\", \"typed_text\": \"\" Goal: What is the average speed of a car?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/45939_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6015, 0.4777]\", \"lift_point\": \"[0.6015, 0.4777]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8836, 0.6107]\", \"lift_point\": \"[0.8836, 0.6107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Barcelona\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6015, 0.4777]\", \"lift_point\": \"[0.6015, 0.4777]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Barcelona>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46158_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8961, 0.4668]\", \"lift_point\": \"[0.8961, 0.4668]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3922, 0.5931]\", \"lift_point\": \"[0.3922, 0.5931]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3922, 0.5931]\", \"lift_point\": \"[0.3922, 0.5931]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3922, 0.5931]\", \"lift_point\": \"[0.3922, 0.5931]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3922, 0.5931]\", \"lift_point\": \"[0.3922, 0.5931]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3578, 0.4981]\", \"lift_point\": \"[0.3578, 0.4981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3922, 0.5931]\", \"lift_point\": \"[0.3922, 0.5931]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3578, 0.4981]\", \"lift_point\": \"[0.3578, 0.4981]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46312_18.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2878, 0.462]\", \"lift_point\": \"[0.2878, 0.462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2878, 0.462]\", \"lift_point\": \"[0.2878, 0.462]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2878, 0.462]\", \"lift_point\": \"[0.2878, 0.462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2878, 0.462]\", \"lift_point\": \"[0.2878, 0.462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2878, 0.462]\", \"lift_point\": \"[0.2878, 0.462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8773, 0.7394]\", \"lift_point\": \"[0.8773, 0.7394]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2878, 0.462]\", \"lift_point\": \"[0.2878, 0.462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.4417]\", \"lift_point\": \"[0.7692, 0.4417]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Chicago\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2878, 0.462]\", \"lift_point\": \"[0.2878, 0.462]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.4417]\", \"lift_point\": \"[0.7692, 0.4417]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46336_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46389_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46389_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"barbados news\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46389_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"barbados news\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.3399]\", \"lift_point\": \"[0.1423, 0.3399]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46389_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"barbados news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.3399]\", \"lift_point\": \"[0.1423, 0.3399]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.227, 0.1974]\", \"lift_point\": \"[0.227, 0.1974]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46389_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"barbados news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.3399]\", \"lift_point\": \"[0.1423, 0.3399]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.227, 0.1974]\", \"lift_point\": \"[0.227, 0.1974]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5472, 0.2965]\", \"lift_point\": \"[0.5472, 0.2965]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46389_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8906, 0.3568]\", \"lift_point\": \"[0.8906, 0.3568]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"barbados news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.3399]\", \"lift_point\": \"[0.1423, 0.3399]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.227, 0.1974]\", \"lift_point\": \"[0.227, 0.1974]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5472, 0.2965]\", \"lift_point\": \"[0.5472, 0.2965]\", \"typed_text\": \"\" Goal: What's the news in Barbados?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46389_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather app\" Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.3821]\", \"lift_point\": \"[0.1463, 0.3821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.3821]\", \"lift_point\": \"[0.1463, 0.3821]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2847, 0.3017]\", \"lift_point\": \"[0.2847, 0.3017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.3821]\", \"lift_point\": \"[0.1463, 0.3821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2847, 0.3017]\", \"lift_point\": \"[0.2847, 0.3017]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.4831]\", \"lift_point\": \"[0.8974, 0.4831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.3821]\", \"lift_point\": \"[0.1463, 0.3821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2847, 0.3017]\", \"lift_point\": \"[0.2847, 0.3017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.4831]\", \"lift_point\": \"[0.8974, 0.4831]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.909, 0.527]\", \"lift_point\": \"[0.909, 0.527]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7747, 0.4858]\", \"lift_point\": \"[0.7747, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.3821]\", \"lift_point\": \"[0.1463, 0.3821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2847, 0.3017]\", \"lift_point\": \"[0.2847, 0.3017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.4831]\", \"lift_point\": \"[0.8974, 0.4831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.909, 0.527]\", \"lift_point\": \"[0.909, 0.527]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8976, 0.2897]\", \"lift_point\": \"[0.8976, 0.2897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0793, 0.4717]\", \"lift_point\": \"[0.0793, 0.4717]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0688, 0.9091]\", \"lift_point\": \"[0.0688, 0.9091]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1463, 0.3821]\", \"lift_point\": \"[0.1463, 0.3821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2847, 0.3017]\", \"lift_point\": \"[0.2847, 0.3017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8974, 0.4831]\", \"lift_point\": \"[0.8974, 0.4831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.909, 0.527]\", \"lift_point\": \"[0.909, 0.527]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8976, 0.2897]\", \"lift_point\": \"[0.8976, 0.2897]\", \"typed_text\": \"\" Goal: Install the Weather app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46406_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest \" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2733, 0.4317]\", \"lift_point\": \"[0.2733, 0.4317]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2733, 0.4317]\", \"lift_point\": \"[0.2733, 0.4317]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6855]\", \"lift_point\": \"[0.7808, 0.6855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2733, 0.4317]\", \"lift_point\": \"[0.2733, 0.4317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0858, 0.5294]\", \"lift_point\": \"[0.0858, 0.5294]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2733, 0.4317]\", \"lift_point\": \"[0.2733, 0.4317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2733, 0.4317]\", \"lift_point\": \"[0.2733, 0.4317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.1127]\", \"lift_point\": \"[0.0478, 0.1127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1449, 0.2732]\", \"lift_point\": \"[0.1449, 0.2732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2162, 0.1768]\", \"lift_point\": \"[0.2162, 0.1768]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2733, 0.4317]\", \"lift_point\": \"[0.2733, 0.4317]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0478, 0.1127]\", \"lift_point\": \"[0.0478, 0.1127]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46436_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in Tokyo?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.7008]\", \"lift_point\": \"[0.7799, 0.7008]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46617_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.7008]\", \"lift_point\": \"[0.7799, 0.7008]\", \"typed_text\": \"\" Goal: What time is it in Tokyo?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.4761]\", \"lift_point\": \"[0.0457, 0.4761]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46617_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.7008]\", \"lift_point\": \"[0.7799, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.4761]\", \"lift_point\": \"[0.0457, 0.4761]\", \"typed_text\": \"\" Goal: What time is it in Tokyo?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Tokyo?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46617_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.7008]\", \"lift_point\": \"[0.7799, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.4761]\", \"lift_point\": \"[0.0457, 0.4761]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Tokyo?\" Goal: What time is it in Tokyo?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1287, 0.2907]\", \"lift_point\": \"[0.1287, 0.2907]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46617_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.7008]\", \"lift_point\": \"[0.7799, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.4761]\", \"lift_point\": \"[0.0457, 0.4761]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Tokyo?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1287, 0.2907]\", \"lift_point\": \"[0.1287, 0.2907]\", \"typed_text\": \"\" Goal: What time is it in Tokyo?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3168, 0.2925]\", \"lift_point\": \"[0.3168, 0.2925]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46617_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7799, 0.7008]\", \"lift_point\": \"[0.7799, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0457, 0.4761]\", \"lift_point\": \"[0.0457, 0.4761]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in Tokyo?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1287, 0.2907]\", \"lift_point\": \"[0.1287, 0.2907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3168, 0.2925]\", \"lift_point\": \"[0.3168, 0.2925]\", \"typed_text\": \"\" Goal: What time is it in Tokyo?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46617_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calendar app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46747_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46747_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3635, 0.1318]\", \"lift_point\": \"[0.3635, 0.1318]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46747_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3635, 0.1318]\", \"lift_point\": \"[0.3635, 0.1318]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9054, 0.9204]\", \"lift_point\": \"[0.9054, 0.9204]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46747_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3635, 0.1318]\", \"lift_point\": \"[0.3635, 0.1318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9054, 0.9204]\", \"lift_point\": \"[0.9054, 0.9204]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.4917]\", \"lift_point\": \"[0.872, 0.4917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46747_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3635, 0.1318]\", \"lift_point\": \"[0.3635, 0.1318]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9054, 0.9204]\", \"lift_point\": \"[0.9054, 0.9204]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.872, 0.4917]\", \"lift_point\": \"[0.872, 0.4917]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46747_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.92, 0.4885]\", \"lift_point\": \"[0.92, 0.4885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.92, 0.4885]\", \"lift_point\": \"[0.92, 0.4885]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8987, 0.4942]\", \"lift_point\": \"[0.8987, 0.4942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.92, 0.4885]\", \"lift_point\": \"[0.92, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8987, 0.4942]\", \"lift_point\": \"[0.8987, 0.4942]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.832, 0.8329]\", \"lift_point\": \"[0.832, 0.8329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.92, 0.4885]\", \"lift_point\": \"[0.92, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8987, 0.4942]\", \"lift_point\": \"[0.8987, 0.4942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.832, 0.8329]\", \"lift_point\": \"[0.832, 0.8329]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5372, 0.8033]\", \"lift_point\": \"[0.5372, 0.8033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.92, 0.4885]\", \"lift_point\": \"[0.92, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8987, 0.4942]\", \"lift_point\": \"[0.8987, 0.4942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.832, 0.8329]\", \"lift_point\": \"[0.832, 0.8329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5372, 0.8033]\", \"lift_point\": \"[0.5372, 0.8033]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.1125]\", \"lift_point\": \"[0.0638, 0.1125]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.92, 0.4885]\", \"lift_point\": \"[0.92, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8987, 0.4942]\", \"lift_point\": \"[0.8987, 0.4942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.832, 0.8329]\", \"lift_point\": \"[0.832, 0.8329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5372, 0.8033]\", \"lift_point\": \"[0.5372, 0.8033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.1125]\", \"lift_point\": \"[0.0638, 0.1125]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7591, 0.2515]\", \"lift_point\": \"[0.7591, 0.2515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4979, 0.4889]\", \"lift_point\": \"[0.4979, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.92, 0.4885]\", \"lift_point\": \"[0.92, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8987, 0.4942]\", \"lift_point\": \"[0.8987, 0.4942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.832, 0.8329]\", \"lift_point\": \"[0.832, 0.8329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5372, 0.8033]\", \"lift_point\": \"[0.5372, 0.8033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0638, 0.1125]\", \"lift_point\": \"[0.0638, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7591, 0.2515]\", \"lift_point\": \"[0.7591, 0.2515]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46805_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.4339]\", \"lift_point\": \"[0.0454, 0.4339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.4339]\", \"lift_point\": \"[0.0454, 0.4339]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.4339]\", \"lift_point\": \"[0.0454, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.4339]\", \"lift_point\": \"[0.0454, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1274, 0.3575]\", \"lift_point\": \"[0.1274, 0.3575]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.4339]\", \"lift_point\": \"[0.0454, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1274, 0.3575]\", \"lift_point\": \"[0.1274, 0.3575]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3573, 0.3028]\", \"lift_point\": \"[0.3573, 0.3028]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.4339]\", \"lift_point\": \"[0.0454, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1274, 0.3575]\", \"lift_point\": \"[0.1274, 0.3575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3573, 0.3028]\", \"lift_point\": \"[0.3573, 0.3028]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8858, 0.821]\", \"lift_point\": \"[0.8858, 0.821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6926]\", \"lift_point\": \"[0.7756, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0454, 0.4339]\", \"lift_point\": \"[0.0454, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1274, 0.3575]\", \"lift_point\": \"[0.1274, 0.3575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3573, 0.3028]\", \"lift_point\": \"[0.3573, 0.3028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8858, 0.821]\", \"lift_point\": \"[0.8858, 0.821]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46834_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6764]\", \"lift_point\": \"[0.7808, 0.6764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6764]\", \"lift_point\": \"[0.7808, 0.6764]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.04, 0.4386]\", \"lift_point\": \"[0.04, 0.4386]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6764]\", \"lift_point\": \"[0.7808, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.04, 0.4386]\", \"lift_point\": \"[0.04, 0.4386]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6764]\", \"lift_point\": \"[0.7808, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.04, 0.4386]\", \"lift_point\": \"[0.04, 0.4386]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung?\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1301, 0.2919]\", \"lift_point\": \"[0.1301, 0.2919]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6764]\", \"lift_point\": \"[0.7808, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.04, 0.4386]\", \"lift_point\": \"[0.04, 0.4386]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1301, 0.2919]\", \"lift_point\": \"[0.1301, 0.2919]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6764]\", \"lift_point\": \"[0.7808, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.04, 0.4386]\", \"lift_point\": \"[0.04, 0.4386]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1301, 0.2919]\", \"lift_point\": \"[0.1301, 0.2919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3269, 0.3957]\", \"lift_point\": \"[0.3269, 0.3957]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6764]\", \"lift_point\": \"[0.7808, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.04, 0.4386]\", \"lift_point\": \"[0.04, 0.4386]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1301, 0.2919]\", \"lift_point\": \"[0.1301, 0.2919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3269, 0.3957]\", \"lift_point\": \"[0.3269, 0.3957]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46842_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calendar app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46859_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46859_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3604, 0.1117]\", \"lift_point\": \"[0.3604, 0.1117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46859_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3604, 0.1117]\", \"lift_point\": \"[0.3604, 0.1117]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9483]\", \"lift_point\": \"[0.8946, 0.9483]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46859_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3604, 0.1117]\", \"lift_point\": \"[0.3604, 0.1117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9483]\", \"lift_point\": \"[0.8946, 0.9483]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9016, 0.9432]\", \"lift_point\": \"[0.9016, 0.9432]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46859_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3604, 0.1117]\", \"lift_point\": \"[0.3604, 0.1117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9483]\", \"lift_point\": \"[0.8946, 0.9483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9016, 0.9432]\", \"lift_point\": \"[0.9016, 0.9432]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8743, 0.5516]\", \"lift_point\": \"[0.8743, 0.5516]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46859_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3604, 0.1117]\", \"lift_point\": \"[0.3604, 0.1117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.9483]\", \"lift_point\": \"[0.8946, 0.9483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9016, 0.9432]\", \"lift_point\": \"[0.9016, 0.9432]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8743, 0.5516]\", \"lift_point\": \"[0.8743, 0.5516]\", \"typed_text\": \"\" Goal: Open the calendar app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46859_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Tokyo to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6793]\", \"lift_point\": \"[0.779, 0.6793]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46931_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6793]\", \"lift_point\": \"[0.779, 0.6793]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.5424]\", \"lift_point\": \"[0.2862, 0.5424]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46931_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6793]\", \"lift_point\": \"[0.779, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.5424]\", \"lift_point\": \"[0.2862, 0.5424]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to Mexico city>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Tokyo to Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46931_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6793]\", \"lift_point\": \"[0.779, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.5424]\", \"lift_point\": \"[0.2862, 0.5424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Tokyo to Mexico city\" Goal: Search for flights from Tokyo to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.7106]\", \"lift_point\": \"[0.1288, 0.7106]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46931_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6793]\", \"lift_point\": \"[0.779, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2862, 0.5424]\", \"lift_point\": \"[0.2862, 0.5424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Tokyo to Mexico city\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.7106]\", \"lift_point\": \"[0.1288, 0.7106]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46931_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46945_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46945_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in NYC>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5679]\", \"lift_point\": \"[0.8927, 0.5679]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46951_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5679]\", \"lift_point\": \"[0.8927, 0.5679]\", \"typed_text\": \"\" Goal: Search for hotels in NYC>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in NYC\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46951_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5679]\", \"lift_point\": \"[0.8927, 0.5679]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in NYC\" Goal: Search for hotels in NYC>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.5119]\", \"lift_point\": \"[0.1215, 0.5119]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46951_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.5679]\", \"lift_point\": \"[0.8927, 0.5679]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in NYC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.5119]\", \"lift_point\": \"[0.1215, 0.5119]\", \"typed_text\": \"\" Goal: Search for hotels in NYC>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/46951_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47094_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47094_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.4029]\", \"lift_point\": \"[0.8946, 0.4029]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47094_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.4029]\", \"lift_point\": \"[0.8946, 0.4029]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47094_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.4029]\", \"lift_point\": \"[0.8946, 0.4029]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1328, 0.376]\", \"lift_point\": \"[0.1328, 0.376]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47094_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8946, 0.4029]\", \"lift_point\": \"[0.8946, 0.4029]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1328, 0.376]\", \"lift_point\": \"[0.1328, 0.376]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47094_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8672, 0.3458]\", \"lift_point\": \"[0.8672, 0.3458]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47147_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8672, 0.3458]\", \"lift_point\": \"[0.8672, 0.3458]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this weekend\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47147_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8672, 0.3458]\", \"lift_point\": \"[0.8672, 0.3458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this weekend\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.3646]\", \"lift_point\": \"[0.1115, 0.3646]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47147_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8672, 0.3458]\", \"lift_point\": \"[0.8672, 0.3458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this weekend\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.3646]\", \"lift_point\": \"[0.1115, 0.3646]\", \"typed_text\": \"\" Goal: What's the news this weekend?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47147_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47166_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47166_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.6502]\", \"lift_point\": \"[0.3575, 0.6502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47166_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.6502]\", \"lift_point\": \"[0.3575, 0.6502]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0786, 0.3897]\", \"lift_point\": \"[0.0786, 0.3897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47166_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3575, 0.6502]\", \"lift_point\": \"[0.3575, 0.6502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0786, 0.3897]\", \"lift_point\": \"[0.0786, 0.3897]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47166_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to San Diego>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.477]\", \"lift_point\": \"[0.8922, 0.477]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47201_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.477]\", \"lift_point\": \"[0.8922, 0.477]\", \"typed_text\": \"\" Goal: Search for flights from NYC to San Diego>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to San Diego\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47201_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.477]\", \"lift_point\": \"[0.8922, 0.477]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to San Diego\" Goal: Search for flights from NYC to San Diego>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1149, 0.5781]\", \"lift_point\": \"[0.1149, 0.5781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47201_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.477]\", \"lift_point\": \"[0.8922, 0.477]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to San Diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1149, 0.5781]\", \"lift_point\": \"[0.1149, 0.5781]\", \"typed_text\": \"\" Goal: Search for flights from NYC to San Diego>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47201_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.9081]\", \"lift_point\": \"[0.0647, 0.9081]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.9081]\", \"lift_point\": \"[0.0647, 0.9081]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Starbucks app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.9081]\", \"lift_point\": \"[0.0647, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Starbucks app\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1382, 0.2541]\", \"lift_point\": \"[0.1382, 0.2541]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.9081]\", \"lift_point\": \"[0.0647, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Starbucks app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1382, 0.2541]\", \"lift_point\": \"[0.1382, 0.2541]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.8067]\", \"lift_point\": \"[0.1424, 0.8067]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.9081]\", \"lift_point\": \"[0.0647, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Starbucks app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1382, 0.2541]\", \"lift_point\": \"[0.1382, 0.2541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.8067]\", \"lift_point\": \"[0.1424, 0.8067]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9035, 0.2342]\", \"lift_point\": \"[0.9035, 0.2342]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.9081]\", \"lift_point\": \"[0.0647, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Starbucks app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1382, 0.2541]\", \"lift_point\": \"[0.1382, 0.2541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.8067]\", \"lift_point\": \"[0.1424, 0.8067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9035, 0.2342]\", \"lift_point\": \"[0.9035, 0.2342]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9013, 0.3565]\", \"lift_point\": \"[0.9013, 0.3565]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.8445]\", \"lift_point\": \"[0.1612, 0.8445]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.7869]\", \"lift_point\": \"[0.0647, 0.7869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.9081]\", \"lift_point\": \"[0.0647, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Starbucks app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1382, 0.2541]\", \"lift_point\": \"[0.1382, 0.2541]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.8067]\", \"lift_point\": \"[0.1424, 0.8067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9035, 0.2342]\", \"lift_point\": \"[0.9035, 0.2342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9013, 0.3565]\", \"lift_point\": \"[0.9013, 0.3565]\", \"typed_text\": \"\" Goal: Install the Starbucks app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47308_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.642]\", \"lift_point\": \"[0.7807, 0.642]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47332_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.642]\", \"lift_point\": \"[0.7807, 0.642]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9372]\", \"lift_point\": \"[0.0606, 0.9372]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47332_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.642]\", \"lift_point\": \"[0.7807, 0.642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9372]\", \"lift_point\": \"[0.0606, 0.9372]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1619, 0.7968]\", \"lift_point\": \"[0.1619, 0.7968]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47332_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7807, 0.642]\", \"lift_point\": \"[0.7807, 0.642]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9372]\", \"lift_point\": \"[0.0606, 0.9372]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1619, 0.7968]\", \"lift_point\": \"[0.1619, 0.7968]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47332_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Peru>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6687]\", \"lift_point\": \"[0.7794, 0.6687]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47382_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6687]\", \"lift_point\": \"[0.7794, 0.6687]\", \"typed_text\": \"\" Goal: Google the capital of Peru>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.371]\", \"lift_point\": \"[0.0562, 0.371]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47382_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6687]\", \"lift_point\": \"[0.7794, 0.6687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.371]\", \"lift_point\": \"[0.0562, 0.371]\", \"typed_text\": \"\" Goal: Google the capital of Peru>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"peru capital\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47382_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6687]\", \"lift_point\": \"[0.7794, 0.6687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.371]\", \"lift_point\": \"[0.0562, 0.371]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"peru capital\" Goal: Google the capital of Peru>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1805, 0.2698]\", \"lift_point\": \"[0.1805, 0.2698]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47382_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6687]\", \"lift_point\": \"[0.7794, 0.6687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.371]\", \"lift_point\": \"[0.0562, 0.371]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"peru capital\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1805, 0.2698]\", \"lift_point\": \"[0.1805, 0.2698]\", \"typed_text\": \"\" Goal: Google the capital of Peru>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47382_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47412_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3743, 0.511]\", \"lift_point\": \"[0.3743, 0.511]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47412_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3743, 0.511]\", \"lift_point\": \"[0.3743, 0.511]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in las vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47412_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3743, 0.511]\", \"lift_point\": \"[0.3743, 0.511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in las vegas\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1275, 0.4238]\", \"lift_point\": \"[0.1275, 0.4238]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47412_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3743, 0.511]\", \"lift_point\": \"[0.3743, 0.511]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in las vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1275, 0.4238]\", \"lift_point\": \"[0.1275, 0.4238]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47412_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7739, 0.7018]\", \"lift_point\": \"[0.7739, 0.7018]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47417_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7739, 0.7018]\", \"lift_point\": \"[0.7739, 0.7018]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.5985]\", \"lift_point\": \"[0.0576, 0.5985]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47417_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7739, 0.7018]\", \"lift_point\": \"[0.7739, 0.7018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.5985]\", \"lift_point\": \"[0.0576, 0.5985]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"buenos aires to seoul flights\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47417_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7739, 0.7018]\", \"lift_point\": \"[0.7739, 0.7018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.5985]\", \"lift_point\": \"[0.0576, 0.5985]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"buenos aires to seoul flights\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.4619]\", \"lift_point\": \"[0.1154, 0.4619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47417_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7739, 0.7018]\", \"lift_point\": \"[0.7739, 0.7018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.5985]\", \"lift_point\": \"[0.0576, 0.5985]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"buenos aires to seoul flights\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.4619]\", \"lift_point\": \"[0.1154, 0.4619]\", \"typed_text\": \"\" Goal: Search for flights from Buenos aires to Seoul>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47417_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4033]\", \"lift_point\": \"[0.0518, 0.4033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4033]\", \"lift_point\": \"[0.0518, 0.4033]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4033]\", \"lift_point\": \"[0.0518, 0.4033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1178, 0.579]\", \"lift_point\": \"[0.1178, 0.579]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4033]\", \"lift_point\": \"[0.0518, 0.4033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1178, 0.579]\", \"lift_point\": \"[0.1178, 0.579]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4033]\", \"lift_point\": \"[0.0518, 0.4033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1178, 0.579]\", \"lift_point\": \"[0.1178, 0.579]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7835, 0.2287]\", \"lift_point\": \"[0.7835, 0.2287]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.7033]\", \"lift_point\": \"[0.7864, 0.7033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0518, 0.4033]\", \"lift_point\": \"[0.0518, 0.4033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent for in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1178, 0.579]\", \"lift_point\": \"[0.1178, 0.579]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7835, 0.2287]\", \"lift_point\": \"[0.7835, 0.2287]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47426_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this afternoon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7712, 0.7138]\", \"lift_point\": \"[0.7712, 0.7138]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47611_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7712, 0.7138]\", \"lift_point\": \"[0.7712, 0.7138]\", \"typed_text\": \"\" Goal: What's the news this afternoon?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.274, 0.2943]\", \"lift_point\": \"[0.274, 0.2943]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47611_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7712, 0.7138]\", \"lift_point\": \"[0.7712, 0.7138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.274, 0.2943]\", \"lift_point\": \"[0.274, 0.2943]\", \"typed_text\": \"\" Goal: What's the news this afternoon?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this afternoon?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47611_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7712, 0.7138]\", \"lift_point\": \"[0.7712, 0.7138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.274, 0.2943]\", \"lift_point\": \"[0.274, 0.2943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this afternoon?\" Goal: What's the news this afternoon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1097, 0.3402]\", \"lift_point\": \"[0.1097, 0.3402]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47611_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7712, 0.7138]\", \"lift_point\": \"[0.7712, 0.7138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.274, 0.2943]\", \"lift_point\": \"[0.274, 0.2943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this afternoon?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1097, 0.3402]\", \"lift_point\": \"[0.1097, 0.3402]\", \"typed_text\": \"\" Goal: What's the news this afternoon?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2323, 0.1328]\", \"lift_point\": \"[0.2323, 0.1328]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47611_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7712, 0.7138]\", \"lift_point\": \"[0.7712, 0.7138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.274, 0.2943]\", \"lift_point\": \"[0.274, 0.2943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this afternoon?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1097, 0.3402]\", \"lift_point\": \"[0.1097, 0.3402]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2323, 0.1328]\", \"lift_point\": \"[0.2323, 0.1328]\", \"typed_text\": \"\" Goal: What's the news this afternoon?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47611_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the downloads>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47717_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47717_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3803, 0.537]\", \"lift_point\": \"[0.3803, 0.537]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47717_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3803, 0.537]\", \"lift_point\": \"[0.3803, 0.537]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47717_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3803, 0.537]\", \"lift_point\": \"[0.3803, 0.537]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.2988]\", \"lift_point\": \"[0.4936, 0.2988]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47717_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3803, 0.537]\", \"lift_point\": \"[0.3803, 0.537]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4936, 0.2988]\", \"lift_point\": \"[0.4936, 0.2988]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47717_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.715]\", \"lift_point\": \"[0.7749, 0.715]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47766_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.715]\", \"lift_point\": \"[0.7749, 0.715]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.2838]\", \"lift_point\": \"[0.0653, 0.2838]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47766_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.715]\", \"lift_point\": \"[0.7749, 0.715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.2838]\", \"lift_point\": \"[0.0653, 0.2838]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the Samsung TV cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47766_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.715]\", \"lift_point\": \"[0.7749, 0.715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.2838]\", \"lift_point\": \"[0.0653, 0.2838]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the Samsung TV cost?\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1114, 0.5949]\", \"lift_point\": \"[0.1114, 0.5949]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47766_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7749, 0.715]\", \"lift_point\": \"[0.7749, 0.715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.2838]\", \"lift_point\": \"[0.0653, 0.2838]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the Samsung TV cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1114, 0.5949]\", \"lift_point\": \"[0.1114, 0.5949]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47766_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7596, 0.6913]\", \"lift_point\": \"[0.7596, 0.6913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47785_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7596, 0.6913]\", \"lift_point\": \"[0.7596, 0.6913]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4368]\", \"lift_point\": \"[0.0679, 0.4368]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47785_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7596, 0.6913]\", \"lift_point\": \"[0.7596, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4368]\", \"lift_point\": \"[0.0679, 0.4368]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Euro\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47785_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7596, 0.6913]\", \"lift_point\": \"[0.7596, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4368]\", \"lift_point\": \"[0.0679, 0.4368]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Euro\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.4518]\", \"lift_point\": \"[0.1213, 0.4518]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47785_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7596, 0.6913]\", \"lift_point\": \"[0.7596, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.4368]\", \"lift_point\": \"[0.0679, 0.4368]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate against the Euro\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.4518]\", \"lift_point\": \"[0.1213, 0.4518]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rate against the Euro?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47785_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47805_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.6962]\", \"lift_point\": \"[0.79, 0.6962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47805_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.6962]\", \"lift_point\": \"[0.79, 0.6962]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.4614]\", \"lift_point\": \"[0.0452, 0.4614]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47805_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.6962]\", \"lift_point\": \"[0.79, 0.6962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.4614]\", \"lift_point\": \"[0.0452, 0.4614]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what is the speed of a jet?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47805_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.6962]\", \"lift_point\": \"[0.79, 0.6962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.4614]\", \"lift_point\": \"[0.0452, 0.4614]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what is the speed of a jet?\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1072, 0.5401]\", \"lift_point\": \"[0.1072, 0.5401]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47805_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.6962]\", \"lift_point\": \"[0.79, 0.6962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0452, 0.4614]\", \"lift_point\": \"[0.0452, 0.4614]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"what is the speed of a jet?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1072, 0.5401]\", \"lift_point\": \"[0.1072, 0.5401]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47805_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1662, 0.8742]\", \"lift_point\": \"[0.1662, 0.8742]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1662, 0.8742]\", \"lift_point\": \"[0.1662, 0.8742]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.8874]\", \"lift_point\": \"[0.0597, 0.8874]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1662, 0.8742]\", \"lift_point\": \"[0.1662, 0.8742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.8874]\", \"lift_point\": \"[0.0597, 0.8874]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0978]\", \"lift_point\": \"[0.0679, 0.0978]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1662, 0.8742]\", \"lift_point\": \"[0.1662, 0.8742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.8874]\", \"lift_point\": \"[0.0597, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0978]\", \"lift_point\": \"[0.0679, 0.0978]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8124, 0.1237]\", \"lift_point\": \"[0.8124, 0.1237]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7684, 0.1235]\", \"lift_point\": \"[0.7684, 0.1235]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1662, 0.8742]\", \"lift_point\": \"[0.1662, 0.8742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0597, 0.8874]\", \"lift_point\": \"[0.0597, 0.8874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0978]\", \"lift_point\": \"[0.0679, 0.0978]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8124, 0.1237]\", \"lift_point\": \"[0.8124, 0.1237]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/47866_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.451]\", \"lift_point\": \"[0.3, 0.451]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.451]\", \"lift_point\": \"[0.3, 0.451]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.473, 0.3296]\", \"lift_point\": \"[0.473, 0.3296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.451]\", \"lift_point\": \"[0.3, 0.451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.473, 0.3296]\", \"lift_point\": \"[0.473, 0.3296]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7301, 0.9206]\", \"lift_point\": \"[0.7301, 0.9206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.451]\", \"lift_point\": \"[0.3, 0.451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.473, 0.3296]\", \"lift_point\": \"[0.473, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7301, 0.9206]\", \"lift_point\": \"[0.7301, 0.9206]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2926, 0.9444]\", \"lift_point\": \"[0.2926, 0.9444]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.451]\", \"lift_point\": \"[0.3, 0.451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.473, 0.3296]\", \"lift_point\": \"[0.473, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7301, 0.9206]\", \"lift_point\": \"[0.7301, 0.9206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2926, 0.9444]\", \"lift_point\": \"[0.2926, 0.9444]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7215, 0.9359]\", \"lift_point\": \"[0.7215, 0.9359]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2436, 0.6799]\", \"lift_point\": \"[0.2436, 0.6799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0694, 0.373]\", \"lift_point\": \"[0.0694, 0.373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HisenseTV cost\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.451]\", \"lift_point\": \"[0.3, 0.451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.473, 0.3296]\", \"lift_point\": \"[0.473, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7301, 0.9206]\", \"lift_point\": \"[0.7301, 0.9206]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2926, 0.9444]\", \"lift_point\": \"[0.2926, 0.9444]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7215, 0.9359]\", \"lift_point\": \"[0.7215, 0.9359]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48109_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.7172]\", \"lift_point\": \"[0.0601, 0.7172]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"cheesecake factory\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2005, 0.3387]\", \"lift_point\": \"[0.2005, 0.3387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7599, 0.2511]\", \"lift_point\": \"[0.7599, 0.2511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.475, 0.4932]\", \"lift_point\": \"[0.475, 0.4932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.3853]\", \"lift_point\": \"[0.783, 0.3853]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.3853]\", \"lift_point\": \"[0.783, 0.3853]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.3853]\", \"lift_point\": \"[0.783, 0.3853]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.3853]\", \"lift_point\": \"[0.783, 0.3853]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4748, 0.6991]\", \"lift_point\": \"[0.4748, 0.6991]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.3853]\", \"lift_point\": \"[0.783, 0.3853]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4748, 0.6991]\", \"lift_point\": \"[0.4748, 0.6991]\", \"typed_text\": \"\" Goal: What's on the menu at Cheesecake Factory?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48659_20.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5077, 0.6172]\", \"lift_point\": \"[0.5077, 0.6172]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5077, 0.6172]\", \"lift_point\": \"[0.5077, 0.6172]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4417, 0.0512]\", \"lift_point\": \"[0.4417, 0.0512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5077, 0.6172]\", \"lift_point\": \"[0.5077, 0.6172]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4417, 0.0512]\", \"lift_point\": \"[0.4417, 0.0512]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4518, 0.0765]\", \"lift_point\": \"[0.4518, 0.0765]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7808, 0.6616]\", \"lift_point\": \"[0.7808, 0.6616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5077, 0.6172]\", \"lift_point\": \"[0.5077, 0.6172]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4417, 0.0512]\", \"lift_point\": \"[0.4417, 0.0512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4518, 0.0765]\", \"lift_point\": \"[0.4518, 0.0765]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3546]\", \"lift_point\": \"[0.062, 0.3546]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5077, 0.6172]\", \"lift_point\": \"[0.5077, 0.6172]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4417, 0.0512]\", \"lift_point\": \"[0.4417, 0.0512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4518, 0.0765]\", \"lift_point\": \"[0.4518, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8004, 0.507]\", \"lift_point\": \"[0.8004, 0.507]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu on ihop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2902]\", \"lift_point\": \"[0.1263, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5077, 0.6172]\", \"lift_point\": \"[0.5077, 0.6172]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4417, 0.0512]\", \"lift_point\": \"[0.4417, 0.0512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4518, 0.0765]\", \"lift_point\": \"[0.4518, 0.0765]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8004, 0.507]\", \"lift_point\": \"[0.8004, 0.507]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48773_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8666, 0.3262]\", \"lift_point\": \"[0.8666, 0.3262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8792, 0.1628]\", \"lift_point\": \"[0.8792, 0.1628]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6863]\", \"lift_point\": \"[0.7715, 0.6863]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.2889]\", \"lift_point\": \"[0.0703, 0.2889]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in malaysia\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.371]\", \"lift_point\": \"[0.1339, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6086, 0.3156]\", \"lift_point\": \"[0.6086, 0.3156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5415, 0.6211]\", \"lift_point\": \"[0.5415, 0.6211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.439]\", \"lift_point\": \"[0.0611, 0.439]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.439]\", \"lift_point\": \"[0.0611, 0.439]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia news\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.439]\", \"lift_point\": \"[0.0611, 0.439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia news\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.439]\", \"lift_point\": \"[0.0611, 0.439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia news\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.2361]\", \"lift_point\": \"[0.1176, 0.2361]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.439]\", \"lift_point\": \"[0.0611, 0.439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia news\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.2361]\", \"lift_point\": \"[0.1176, 0.2361]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.3016]\", \"lift_point\": \"[0.8276, 0.3016]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0611, 0.439]\", \"lift_point\": \"[0.0611, 0.439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia news\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.2361]\", \"lift_point\": \"[0.1176, 0.2361]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8276, 0.3016]\", \"lift_point\": \"[0.8276, 0.3016]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48850_38.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new bronzer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\" Goal: Search for a new bronzer>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.3188]\", \"lift_point\": \"[0.0715, 0.3188]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.3188]\", \"lift_point\": \"[0.0715, 0.3188]\", \"typed_text\": \"\" Goal: Search for a new bronzer>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new bronzer\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.3188]\", \"lift_point\": \"[0.0715, 0.3188]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new bronzer\" Goal: Search for a new bronzer>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.3188]\", \"lift_point\": \"[0.0715, 0.3188]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new bronzer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new bronzer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1267, 0.2447]\", \"lift_point\": \"[0.1267, 0.2447]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.3188]\", \"lift_point\": \"[0.0715, 0.3188]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new bronzer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1267, 0.2447]\", \"lift_point\": \"[0.1267, 0.2447]\", \"typed_text\": \"\" Goal: Search for a new bronzer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.3188]\", \"lift_point\": \"[0.0715, 0.3188]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new bronzer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1267, 0.2447]\", \"lift_point\": \"[0.1267, 0.2447]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new bronzer>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.856, 0.2838]\", \"lift_point\": \"[0.856, 0.2838]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.6764]\", \"lift_point\": \"[0.7791, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0715, 0.3188]\", \"lift_point\": \"[0.0715, 0.3188]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new bronzer\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1267, 0.2447]\", \"lift_point\": \"[0.1267, 0.2447]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.856, 0.2838]\", \"lift_point\": \"[0.856, 0.2838]\", \"typed_text\": \"\" Goal: Search for a new bronzer>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48873_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.3311]\", \"lift_point\": \"[0.3063, 0.3311]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.3311]\", \"lift_point\": \"[0.3063, 0.3311]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5192, 0.696]\", \"lift_point\": \"[0.5192, 0.696]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.3311]\", \"lift_point\": \"[0.3063, 0.3311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5192, 0.696]\", \"lift_point\": \"[0.5192, 0.696]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5091, 0.7017]\", \"lift_point\": \"[0.5091, 0.7017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.3311]\", \"lift_point\": \"[0.3063, 0.3311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5192, 0.696]\", \"lift_point\": \"[0.5192, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5091, 0.7017]\", \"lift_point\": \"[0.5091, 0.7017]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4188, 0.5085]\", \"lift_point\": \"[0.4188, 0.5085]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.3311]\", \"lift_point\": \"[0.3063, 0.3311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5192, 0.696]\", \"lift_point\": \"[0.5192, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5091, 0.7017]\", \"lift_point\": \"[0.5091, 0.7017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4188, 0.5085]\", \"lift_point\": \"[0.4188, 0.5085]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6857, 0.7309]\", \"lift_point\": \"[0.6857, 0.7309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.375, 0.88]\", \"lift_point\": \"[0.375, 0.88]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.085, 0.1138]\", \"lift_point\": \"[0.085, 0.1138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6539, 0.8814]\", \"lift_point\": \"[0.6539, 0.8814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3063, 0.3311]\", \"lift_point\": \"[0.3063, 0.3311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5192, 0.696]\", \"lift_point\": \"[0.5192, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5091, 0.7017]\", \"lift_point\": \"[0.5091, 0.7017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4188, 0.5085]\", \"lift_point\": \"[0.4188, 0.5085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6857, 0.7309]\", \"lift_point\": \"[0.6857, 0.7309]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48893_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach towel\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach towel\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.352]\", \"lift_point\": \"[0.1208, 0.352]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach towel\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.352]\", \"lift_point\": \"[0.1208, 0.352]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3398, 0.2676]\", \"lift_point\": \"[0.3398, 0.2676]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach towel\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.352]\", \"lift_point\": \"[0.1208, 0.352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3398, 0.2676]\", \"lift_point\": \"[0.3398, 0.2676]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach towel\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.352]\", \"lift_point\": \"[0.1208, 0.352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3398, 0.2676]\", \"lift_point\": \"[0.3398, 0.2676]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3445, 0.7999]\", \"lift_point\": \"[0.3445, 0.7999]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach towel\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.352]\", \"lift_point\": \"[0.1208, 0.352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3398, 0.2676]\", \"lift_point\": \"[0.3398, 0.2676]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3445, 0.7999]\", \"lift_point\": \"[0.3445, 0.7999]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8888, 0.4811]\", \"lift_point\": \"[0.8888, 0.4811]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6763]\", \"lift_point\": \"[0.7851, 0.6763]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2925, 0.5378]\", \"lift_point\": \"[0.2925, 0.5378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice beach towel\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1208, 0.352]\", \"lift_point\": \"[0.1208, 0.352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3398, 0.2676]\", \"lift_point\": \"[0.3398, 0.2676]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3445, 0.7999]\", \"lift_point\": \"[0.3445, 0.7999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8888, 0.4811]\", \"lift_point\": \"[0.8888, 0.4811]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/48952_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.104]\", \"lift_point\": \"[0.8002, 0.104]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.104]\", \"lift_point\": \"[0.8002, 0.104]\", \"typed_text\": \"\" Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.4905]\", \"lift_point\": \"[0.0605, 0.4905]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.104]\", \"lift_point\": \"[0.8002, 0.104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.4905]\", \"lift_point\": \"[0.0605, 0.4905]\", \"typed_text\": \"\" Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"wikipedia app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.104]\", \"lift_point\": \"[0.8002, 0.104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.4905]\", \"lift_point\": \"[0.0605, 0.4905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"wikipedia app\" Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.258]\", \"lift_point\": \"[0.1288, 0.258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.104]\", \"lift_point\": \"[0.8002, 0.104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.4905]\", \"lift_point\": \"[0.0605, 0.4905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"wikipedia app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.258]\", \"lift_point\": \"[0.1288, 0.258]\", \"typed_text\": \"\" Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1495, 0.8624]\", \"lift_point\": \"[0.1495, 0.8624]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.104]\", \"lift_point\": \"[0.8002, 0.104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.4905]\", \"lift_point\": \"[0.0605, 0.4905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"wikipedia app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.258]\", \"lift_point\": \"[0.1288, 0.258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1495, 0.8624]\", \"lift_point\": \"[0.1495, 0.8624]\", \"typed_text\": \"\" Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.149, 0.8472]\", \"lift_point\": \"[0.149, 0.8472]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.104]\", \"lift_point\": \"[0.8002, 0.104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.4905]\", \"lift_point\": \"[0.0605, 0.4905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"wikipedia app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1288, 0.258]\", \"lift_point\": \"[0.1288, 0.258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1495, 0.8624]\", \"lift_point\": \"[0.1495, 0.8624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.149, 0.8472]\", \"lift_point\": \"[0.149, 0.8472]\", \"typed_text\": \"\" Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49119_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49366_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8228, 0.5871]\", \"lift_point\": \"[0.8228, 0.5871]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49366_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8228, 0.5871]\", \"lift_point\": \"[0.8228, 0.5871]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.068, 0.4302]\", \"lift_point\": \"[0.068, 0.4302]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49366_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8228, 0.5871]\", \"lift_point\": \"[0.8228, 0.5871]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.068, 0.4302]\", \"lift_point\": \"[0.068, 0.4302]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49366_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8228, 0.5871]\", \"lift_point\": \"[0.8228, 0.5871]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.068, 0.4302]\", \"lift_point\": \"[0.068, 0.4302]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49366_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8228, 0.5871]\", \"lift_point\": \"[0.8228, 0.5871]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.068, 0.4302]\", \"lift_point\": \"[0.068, 0.4302]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2442, 0.1528]\", \"lift_point\": \"[0.2442, 0.1528]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49366_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8228, 0.5871]\", \"lift_point\": \"[0.8228, 0.5871]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.068, 0.4302]\", \"lift_point\": \"[0.068, 0.4302]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2442, 0.1528]\", \"lift_point\": \"[0.2442, 0.1528]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49366_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.8915]\", \"lift_point\": \"[0.1785, 0.8915]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.8915]\", \"lift_point\": \"[0.1785, 0.8915]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power+ Snow Blower?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.8915]\", \"lift_point\": \"[0.1785, 0.8915]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power+ Snow Blower?\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.8915]\", \"lift_point\": \"[0.1785, 0.8915]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power+ Snow Blower?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2495, 0.0452]\", \"lift_point\": \"[0.2495, 0.0452]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.8915]\", \"lift_point\": \"[0.1785, 0.8915]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power+ Snow Blower?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2495, 0.0452]\", \"lift_point\": \"[0.2495, 0.0452]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.8915]\", \"lift_point\": \"[0.1785, 0.8915]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power+ Snow Blower?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2495, 0.0452]\", \"lift_point\": \"[0.2495, 0.0452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3674, 0.3126]\", \"lift_point\": \"[0.3674, 0.3126]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8145, 0.6085]\", \"lift_point\": \"[0.8145, 0.6085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1785, 0.8915]\", \"lift_point\": \"[0.1785, 0.8915]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the 1000-Watt EGO Power+ Snow Blower?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2495, 0.0452]\", \"lift_point\": \"[0.2495, 0.0452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3674, 0.3126]\", \"lift_point\": \"[0.3674, 0.3126]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49383_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8218, 0.615]\", \"lift_point\": \"[0.8218, 0.615]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49409_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8218, 0.615]\", \"lift_point\": \"[0.8218, 0.615]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7961, 0.5846]\", \"lift_point\": \"[0.7961, 0.5846]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49409_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8218, 0.615]\", \"lift_point\": \"[0.8218, 0.615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7961, 0.5846]\", \"lift_point\": \"[0.7961, 0.5846]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.18, 0.7055]\", \"lift_point\": \"[0.18, 0.7055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49409_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8218, 0.615]\", \"lift_point\": \"[0.8218, 0.615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7961, 0.5846]\", \"lift_point\": \"[0.7961, 0.5846]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.18, 0.7055]\", \"lift_point\": \"[0.18, 0.7055]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.114, 0.9439]\", \"lift_point\": \"[0.114, 0.9439]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49409_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8218, 0.615]\", \"lift_point\": \"[0.8218, 0.615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7961, 0.5846]\", \"lift_point\": \"[0.7961, 0.5846]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.18, 0.7055]\", \"lift_point\": \"[0.18, 0.7055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.114, 0.9439]\", \"lift_point\": \"[0.114, 0.9439]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49409_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8218, 0.615]\", \"lift_point\": \"[0.8218, 0.615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7961, 0.5846]\", \"lift_point\": \"[0.7961, 0.5846]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.18, 0.7055]\", \"lift_point\": \"[0.18, 0.7055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.114, 0.9439]\", \"lift_point\": \"[0.114, 0.9439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.2766]\", \"lift_point\": \"[0.1698, 0.2766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49409_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8218, 0.615]\", \"lift_point\": \"[0.8218, 0.615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7961, 0.5846]\", \"lift_point\": \"[0.7961, 0.5846]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.18, 0.7055]\", \"lift_point\": \"[0.18, 0.7055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.114, 0.9439]\", \"lift_point\": \"[0.114, 0.9439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in seattle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1698, 0.2766]\", \"lift_point\": \"[0.1698, 0.2766]\", \"typed_text\": \"\" Goal: Search for hotels in Seattle>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49409_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.4877]\", \"lift_point\": \"[0.0428, 0.4877]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.4877]\", \"lift_point\": \"[0.0428, 0.4877]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.4877]\", \"lift_point\": \"[0.0428, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.5962]\", \"lift_point\": \"[0.8252, 0.5962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.4877]\", \"lift_point\": \"[0.0428, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5536, 0.3264]\", \"lift_point\": \"[0.5536, 0.3264]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8001, 0.609]\", \"lift_point\": \"[0.8001, 0.609]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0428, 0.4877]\", \"lift_point\": \"[0.0428, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5536, 0.3264]\", \"lift_point\": \"[0.5536, 0.3264]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49441_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8259, 0.57]\", \"lift_point\": \"[0.8259, 0.57]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49446_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8259, 0.57]\", \"lift_point\": \"[0.8259, 0.57]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.5665]\", \"lift_point\": \"[0.0601, 0.5665]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49446_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8259, 0.57]\", \"lift_point\": \"[0.8259, 0.57]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.5665]\", \"lift_point\": \"[0.0601, 0.5665]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Las Vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49446_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8259, 0.57]\", \"lift_point\": \"[0.8259, 0.57]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.5665]\", \"lift_point\": \"[0.0601, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Las Vegas\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1359, 0.2507]\", \"lift_point\": \"[0.1359, 0.2507]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49446_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8259, 0.57]\", \"lift_point\": \"[0.8259, 0.57]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.5665]\", \"lift_point\": \"[0.0601, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1359, 0.2507]\", \"lift_point\": \"[0.1359, 0.2507]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49446_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1688, 0.5554]\", \"lift_point\": \"[0.1688, 0.5554]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1688, 0.5554]\", \"lift_point\": \"[0.1688, 0.5554]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9255]\", \"lift_point\": \"[0.113, 0.9255]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1688, 0.5554]\", \"lift_point\": \"[0.1688, 0.5554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9255]\", \"lift_point\": \"[0.113, 0.9255]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rateagainst the Mexican Peso?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1688, 0.5554]\", \"lift_point\": \"[0.1688, 0.5554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9255]\", \"lift_point\": \"[0.113, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rateagainst the Mexican Peso?\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8325, 0.5994]\", \"lift_point\": \"[0.8325, 0.5994]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1688, 0.5554]\", \"lift_point\": \"[0.1688, 0.5554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9255]\", \"lift_point\": \"[0.113, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rateagainst the Mexican Peso?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2311, 0.077]\", \"lift_point\": \"[0.2311, 0.077]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8179, 0.6065]\", \"lift_point\": \"[0.8179, 0.6065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1688, 0.5554]\", \"lift_point\": \"[0.1688, 0.5554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.9255]\", \"lift_point\": \"[0.113, 0.9255]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats US dollar exchange rateagainst the Mexican Peso?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2311, 0.077]\", \"lift_point\": \"[0.2311, 0.077]\", \"typed_text\": \"\" Goal: What's US dollar exchange rateagainst the Mexican Peso?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49469_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49566_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8395, 0.6186]\", \"lift_point\": \"[0.8395, 0.6186]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49566_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8395, 0.6186]\", \"lift_point\": \"[0.8395, 0.6186]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.569]\", \"lift_point\": \"[0.1732, 0.569]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49566_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8395, 0.6186]\", \"lift_point\": \"[0.8395, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.569]\", \"lift_point\": \"[0.1732, 0.569]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.9264]\", \"lift_point\": \"[0.1094, 0.9264]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49566_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8395, 0.6186]\", \"lift_point\": \"[0.8395, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.569]\", \"lift_point\": \"[0.1732, 0.569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.9264]\", \"lift_point\": \"[0.1094, 0.9264]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49566_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8395, 0.6186]\", \"lift_point\": \"[0.8395, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.569]\", \"lift_point\": \"[0.1732, 0.569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.9264]\", \"lift_point\": \"[0.1094, 0.9264]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49566_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8395, 0.6186]\", \"lift_point\": \"[0.8395, 0.6186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.569]\", \"lift_point\": \"[0.1732, 0.569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1094, 0.9264]\", \"lift_point\": \"[0.1094, 0.9264]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49566_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Chrome app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49691_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Google Chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6154]\", \"lift_point\": \"[0.8171, 0.6154]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49691_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6154]\", \"lift_point\": \"[0.8171, 0.6154]\", \"typed_text\": \"\" Goal: Check the settings for the Google Chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5185, 0.6138]\", \"lift_point\": \"[0.5185, 0.6138]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49691_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6154]\", \"lift_point\": \"[0.8171, 0.6154]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5185, 0.6138]\", \"lift_point\": \"[0.5185, 0.6138]\", \"typed_text\": \"\" Goal: Check the settings for the Google Chrome app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5282, 0.6061]\", \"lift_point\": \"[0.5282, 0.6061]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49691_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8171, 0.6154]\", \"lift_point\": \"[0.8171, 0.6154]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5185, 0.6138]\", \"lift_point\": \"[0.5185, 0.6138]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5282, 0.6061]\", \"lift_point\": \"[0.5282, 0.6061]\", \"typed_text\": \"\" Goal: Check the settings for the Google Chrome app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49691_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the ESPN app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.8166]\", \"lift_point\": \"[0.1419, 0.8166]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.8166]\", \"lift_point\": \"[0.1419, 0.8166]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9164, 0.5564]\", \"lift_point\": \"[0.9164, 0.5564]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.8166]\", \"lift_point\": \"[0.1419, 0.8166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9164, 0.5564]\", \"lift_point\": \"[0.9164, 0.5564]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9183, 0.3832]\", \"lift_point\": \"[0.9183, 0.3832]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.8166]\", \"lift_point\": \"[0.1419, 0.8166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9164, 0.5564]\", \"lift_point\": \"[0.9164, 0.5564]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9183, 0.3832]\", \"lift_point\": \"[0.9183, 0.3832]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1468, 0.8864]\", \"lift_point\": \"[0.1468, 0.8864]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.4037]\", \"lift_point\": \"[0.777, 0.4037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.548]\", \"lift_point\": \"[0.0461, 0.548]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"espn app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.8166]\", \"lift_point\": \"[0.1419, 0.8166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9164, 0.5564]\", \"lift_point\": \"[0.9164, 0.5564]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9183, 0.3832]\", \"lift_point\": \"[0.9183, 0.3832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1468, 0.8864]\", \"lift_point\": \"[0.1468, 0.8864]\", \"typed_text\": \"\" Goal: Install the ESPN app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49800_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.811, 0.6427]\", \"lift_point\": \"[0.811, 0.6427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49984_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.811, 0.6427]\", \"lift_point\": \"[0.811, 0.6427]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1875, 0.5694]\", \"lift_point\": \"[0.1875, 0.5694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49984_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.811, 0.6427]\", \"lift_point\": \"[0.811, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1875, 0.5694]\", \"lift_point\": \"[0.1875, 0.5694]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.9197]\", \"lift_point\": \"[0.1198, 0.9197]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49984_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.811, 0.6427]\", \"lift_point\": \"[0.811, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1875, 0.5694]\", \"lift_point\": \"[0.1875, 0.5694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.9197]\", \"lift_point\": \"[0.1198, 0.9197]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Boston?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49984_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.811, 0.6427]\", \"lift_point\": \"[0.811, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1875, 0.5694]\", \"lift_point\": \"[0.1875, 0.5694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.9197]\", \"lift_point\": \"[0.1198, 0.9197]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Boston?\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49984_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.811, 0.6427]\", \"lift_point\": \"[0.811, 0.6427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1875, 0.5694]\", \"lift_point\": \"[0.1875, 0.5694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.9197]\", \"lift_point\": \"[0.1198, 0.9197]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Boston?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Boston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49984_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8002, 0.6167]\", \"lift_point\": \"[0.8002, 0.6167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.7803]\", \"lift_point\": \"[0.1899, 0.7803]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.7803]\", \"lift_point\": \"[0.1899, 0.7803]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.925]\", \"lift_point\": \"[0.1159, 0.925]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.7803]\", \"lift_point\": \"[0.1899, 0.7803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.925]\", \"lift_point\": \"[0.1159, 0.925]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Argentina?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.7803]\", \"lift_point\": \"[0.1899, 0.7803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.925]\", \"lift_point\": \"[0.1159, 0.925]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Argentina?\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.7803]\", \"lift_point\": \"[0.1899, 0.7803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.925]\", \"lift_point\": \"[0.1159, 0.925]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Argentina?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2411, 0.1894]\", \"lift_point\": \"[0.2411, 0.1894]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7981, 0.6113]\", \"lift_point\": \"[0.7981, 0.6113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1899, 0.7803]\", \"lift_point\": \"[0.1899, 0.7803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.925]\", \"lift_point\": \"[0.1159, 0.925]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Argentina?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2411, 0.1894]\", \"lift_point\": \"[0.2411, 0.1894]\", \"typed_text\": \"\" Goal: What's the news in Argentina?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/49998_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50059_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8132, 0.6673]\", \"lift_point\": \"[0.8132, 0.6673]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50059_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8132, 0.6673]\", \"lift_point\": \"[0.8132, 0.6673]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6064]\", \"lift_point\": \"[0.8006, 0.6064]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50059_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8132, 0.6673]\", \"lift_point\": \"[0.8132, 0.6673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6064]\", \"lift_point\": \"[0.8006, 0.6064]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.9005]\", \"lift_point\": \"[0.1823, 0.9005]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50059_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8132, 0.6673]\", \"lift_point\": \"[0.8132, 0.6673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6064]\", \"lift_point\": \"[0.8006, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.9005]\", \"lift_point\": \"[0.1823, 0.9005]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space technology?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50059_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8132, 0.6673]\", \"lift_point\": \"[0.8132, 0.6673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6064]\", \"lift_point\": \"[0.8006, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.9005]\", \"lift_point\": \"[0.1823, 0.9005]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space technology?\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50059_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8252, 0.6178]\", \"lift_point\": \"[0.8252, 0.6178]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8132, 0.6673]\", \"lift_point\": \"[0.8132, 0.6673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6064]\", \"lift_point\": \"[0.8006, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.9005]\", \"lift_point\": \"[0.1823, 0.9005]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space technology?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50059_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5763, 0.5456]\", \"lift_point\": \"[0.5763, 0.5456]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5763, 0.5456]\", \"lift_point\": \"[0.5763, 0.5456]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.7928]\", \"lift_point\": \"[0.0635, 0.7928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5763, 0.5456]\", \"lift_point\": \"[0.5763, 0.5456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.7928]\", \"lift_point\": \"[0.0635, 0.7928]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5763, 0.5456]\", \"lift_point\": \"[0.5763, 0.5456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.7928]\", \"lift_point\": \"[0.0635, 0.7928]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5044]\", \"lift_point\": \"[0.0606, 0.5044]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5763, 0.5456]\", \"lift_point\": \"[0.5763, 0.5456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.7928]\", \"lift_point\": \"[0.0635, 0.7928]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5044]\", \"lift_point\": \"[0.0606, 0.5044]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5763, 0.5456]\", \"lift_point\": \"[0.5763, 0.5456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.7928]\", \"lift_point\": \"[0.0635, 0.7928]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5044]\", \"lift_point\": \"[0.0606, 0.5044]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5763, 0.5456]\", \"lift_point\": \"[0.5763, 0.5456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.7928]\", \"lift_point\": \"[0.0635, 0.7928]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5044]\", \"lift_point\": \"[0.0606, 0.5044]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50131_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.735, 0.5885]\", \"lift_point\": \"[0.735, 0.5885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.735, 0.5885]\", \"lift_point\": \"[0.735, 0.5885]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0546, 0.7868]\", \"lift_point\": \"[0.0546, 0.7868]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.735, 0.5885]\", \"lift_point\": \"[0.735, 0.5885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0546, 0.7868]\", \"lift_point\": \"[0.0546, 0.7868]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9384]\", \"lift_point\": \"[0.0613, 0.9384]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.735, 0.5885]\", \"lift_point\": \"[0.735, 0.5885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0546, 0.7868]\", \"lift_point\": \"[0.0546, 0.7868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9384]\", \"lift_point\": \"[0.0613, 0.9384]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ebay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.735, 0.5885]\", \"lift_point\": \"[0.735, 0.5885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0546, 0.7868]\", \"lift_point\": \"[0.0546, 0.7868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9384]\", \"lift_point\": \"[0.0613, 0.9384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ebay\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.2809]\", \"lift_point\": \"[0.117, 0.2809]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.735, 0.5885]\", \"lift_point\": \"[0.735, 0.5885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0546, 0.7868]\", \"lift_point\": \"[0.0546, 0.7868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9384]\", \"lift_point\": \"[0.0613, 0.9384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ebay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.2809]\", \"lift_point\": \"[0.117, 0.2809]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1343, 0.9098]\", \"lift_point\": \"[0.1343, 0.9098]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.735, 0.5885]\", \"lift_point\": \"[0.735, 0.5885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0546, 0.7868]\", \"lift_point\": \"[0.0546, 0.7868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0613, 0.9384]\", \"lift_point\": \"[0.0613, 0.9384]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ebay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.2809]\", \"lift_point\": \"[0.117, 0.2809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1343, 0.9098]\", \"lift_point\": \"[0.1343, 0.9098]\", \"typed_text\": \"\" Goal: Install the eBay app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50192_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.7322]\", \"lift_point\": \"[0.782, 0.7322]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.7322]\", \"lift_point\": \"[0.782, 0.7322]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.6694]\", \"lift_point\": \"[0.1816, 0.6694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.7322]\", \"lift_point\": \"[0.782, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.6694]\", \"lift_point\": \"[0.1816, 0.6694]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9314]\", \"lift_point\": \"[0.1082, 0.9314]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.7322]\", \"lift_point\": \"[0.782, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.6694]\", \"lift_point\": \"[0.1816, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9314]\", \"lift_point\": \"[0.1082, 0.9314]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.7322]\", \"lift_point\": \"[0.782, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.6694]\", \"lift_point\": \"[0.1816, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9314]\", \"lift_point\": \"[0.1082, 0.9314]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.7322]\", \"lift_point\": \"[0.782, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.6694]\", \"lift_point\": \"[0.1816, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9314]\", \"lift_point\": \"[0.1082, 0.9314]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2264, 0.1648]\", \"lift_point\": \"[0.2264, 0.1648]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.782, 0.7322]\", \"lift_point\": \"[0.782, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1816, 0.6694]\", \"lift_point\": \"[0.1816, 0.6694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1082, 0.9314]\", \"lift_point\": \"[0.1082, 0.9314]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Colombia?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2264, 0.1648]\", \"lift_point\": \"[0.2264, 0.1648]\", \"typed_text\": \"\" Goal: What's the news in Colombia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50308_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6008, 0.1417]\", \"lift_point\": \"[0.6008, 0.1417]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6008, 0.1417]\", \"lift_point\": \"[0.6008, 0.1417]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1195, 0.6781]\", \"lift_point\": \"[0.1195, 0.6781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6008, 0.1417]\", \"lift_point\": \"[0.6008, 0.1417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1195, 0.6781]\", \"lift_point\": \"[0.1195, 0.6781]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9191]\", \"lift_point\": \"[0.0589, 0.9191]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6008, 0.1417]\", \"lift_point\": \"[0.6008, 0.1417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1195, 0.6781]\", \"lift_point\": \"[0.1195, 0.6781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9191]\", \"lift_point\": \"[0.0589, 0.9191]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6008, 0.1417]\", \"lift_point\": \"[0.6008, 0.1417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1195, 0.6781]\", \"lift_point\": \"[0.1195, 0.6781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9191]\", \"lift_point\": \"[0.0589, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6008, 0.1417]\", \"lift_point\": \"[0.6008, 0.1417]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1195, 0.6781]\", \"lift_point\": \"[0.1195, 0.6781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9191]\", \"lift_point\": \"[0.0589, 0.9191]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50344_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.8981]\", \"lift_point\": \"[0.0645, 0.8981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.8981]\", \"lift_point\": \"[0.0645, 0.8981]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.8981]\", \"lift_point\": \"[0.0645, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1357, 0.7171]\", \"lift_point\": \"[0.1357, 0.7171]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.8981]\", \"lift_point\": \"[0.0645, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1357, 0.7171]\", \"lift_point\": \"[0.1357, 0.7171]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1837, 0.2179]\", \"lift_point\": \"[0.1837, 0.2179]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.8981]\", \"lift_point\": \"[0.0645, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1357, 0.7171]\", \"lift_point\": \"[0.1357, 0.7171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1837, 0.2179]\", \"lift_point\": \"[0.1837, 0.2179]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.4599]\", \"lift_point\": \"[0.8764, 0.4599]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.8981]\", \"lift_point\": \"[0.0645, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1357, 0.7171]\", \"lift_point\": \"[0.1357, 0.7171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1837, 0.2179]\", \"lift_point\": \"[0.1837, 0.2179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.4599]\", \"lift_point\": \"[0.8764, 0.4599]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8752, 0.831]\", \"lift_point\": \"[0.8752, 0.831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5493, 0.8544]\", \"lift_point\": \"[0.5493, 0.8544]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1512, 0.7378]\", \"lift_point\": \"[0.1512, 0.7378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.8981]\", \"lift_point\": \"[0.0645, 0.8981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1357, 0.7171]\", \"lift_point\": \"[0.1357, 0.7171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1837, 0.2179]\", \"lift_point\": \"[0.1837, 0.2179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8764, 0.4599]\", \"lift_point\": \"[0.8764, 0.4599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8752, 0.831]\", \"lift_point\": \"[0.8752, 0.831]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50434_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1205, 0.4479]\", \"lift_point\": \"[0.1205, 0.4479]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7901, 0.6849]\", \"lift_point\": \"[0.7901, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1205, 0.4479]\", \"lift_point\": \"[0.1205, 0.4479]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4429, 0.4461]\", \"lift_point\": \"[0.4429, 0.4461]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1205, 0.4479]\", \"lift_point\": \"[0.1205, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4429, 0.4461]\", \"lift_point\": \"[0.4429, 0.4461]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.4827]\", \"lift_point\": \"[0.8925, 0.4827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6923]\", \"lift_point\": \"[0.7864, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1205, 0.4479]\", \"lift_point\": \"[0.1205, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4429, 0.4461]\", \"lift_point\": \"[0.4429, 0.4461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.4827]\", \"lift_point\": \"[0.8925, 0.4827]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.5485]\", \"lift_point\": \"[0.8963, 0.5485]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7889, 0.6923]\", \"lift_point\": \"[0.7889, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1205, 0.4479]\", \"lift_point\": \"[0.1205, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4429, 0.4461]\", \"lift_point\": \"[0.4429, 0.4461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.4827]\", \"lift_point\": \"[0.8925, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.5485]\", \"lift_point\": \"[0.8963, 0.5485]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9168, 0.9339]\", \"lift_point\": \"[0.9168, 0.9339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.7127]\", \"lift_point\": \"[0.0479, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1205, 0.4479]\", \"lift_point\": \"[0.1205, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4429, 0.4461]\", \"lift_point\": \"[0.4429, 0.4461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.4827]\", \"lift_point\": \"[0.8925, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.5485]\", \"lift_point\": \"[0.8963, 0.5485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9168, 0.9339]\", \"lift_point\": \"[0.9168, 0.9339]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8965, 0.5618]\", \"lift_point\": \"[0.8965, 0.5618]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0463, 0.9337]\", \"lift_point\": \"[0.0463, 0.9337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in theFalkland Islands\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1205, 0.4479]\", \"lift_point\": \"[0.1205, 0.4479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4429, 0.4461]\", \"lift_point\": \"[0.4429, 0.4461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.4827]\", \"lift_point\": \"[0.8925, 0.4827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8963, 0.5485]\", \"lift_point\": \"[0.8963, 0.5485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9168, 0.9339]\", \"lift_point\": \"[0.9168, 0.9339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8965, 0.5618]\", \"lift_point\": \"[0.8965, 0.5618]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50661_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7859, 0.6832]\", \"lift_point\": \"[0.7859, 0.6832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7944, 0.6849]\", \"lift_point\": \"[0.7944, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Canada\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6849]\", \"lift_point\": \"[0.779, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Canada\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1193, 0.3119]\", \"lift_point\": \"[0.1193, 0.3119]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7839, 0.6849]\", \"lift_point\": \"[0.7839, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Canada\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1193, 0.3119]\", \"lift_point\": \"[0.1193, 0.3119]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.2934]\", \"lift_point\": \"[0.1281, 0.2934]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.6927]\", \"lift_point\": \"[0.7913, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Canada\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1193, 0.3119]\", \"lift_point\": \"[0.1193, 0.3119]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.2934]\", \"lift_point\": \"[0.1281, 0.2934]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.23]\", \"lift_point\": \"[0.1233, 0.23]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7949, 0.6927]\", \"lift_point\": \"[0.7949, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Canada\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1193, 0.3119]\", \"lift_point\": \"[0.1193, 0.3119]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.2934]\", \"lift_point\": \"[0.1281, 0.2934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.23]\", \"lift_point\": \"[0.1233, 0.23]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.3334]\", \"lift_point\": \"[0.117, 0.3334]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.6087]\", \"lift_point\": \"[0.0561, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Canada\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1193, 0.3119]\", \"lift_point\": \"[0.1193, 0.3119]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.2934]\", \"lift_point\": \"[0.1281, 0.2934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.23]\", \"lift_point\": \"[0.1233, 0.23]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.3334]\", \"lift_point\": \"[0.117, 0.3334]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.1249]\", \"lift_point\": \"[0.2387, 0.1249]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.9348]\", \"lift_point\": \"[0.0578, 0.9348]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Canada\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1193, 0.3119]\", \"lift_point\": \"[0.1193, 0.3119]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.2934]\", \"lift_point\": \"[0.1281, 0.2934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1233, 0.23]\", \"lift_point\": \"[0.1233, 0.23]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.117, 0.3334]\", \"lift_point\": \"[0.117, 0.3334]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2387, 0.1249]\", \"lift_point\": \"[0.2387, 0.1249]\", \"typed_text\": \"\" Goal: Google the capital of Canada>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50688_16.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1606, 0.1203]\", \"lift_point\": \"[0.1606, 0.1203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1606, 0.1203]\", \"lift_point\": \"[0.1606, 0.1203]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1606, 0.1203]\", \"lift_point\": \"[0.1606, 0.1203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4868, 0.873]\", \"lift_point\": \"[0.4868, 0.873]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1606, 0.1203]\", \"lift_point\": \"[0.1606, 0.1203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4868, 0.873]\", \"lift_point\": \"[0.4868, 0.873]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.3059]\", \"lift_point\": \"[0.0657, 0.3059]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1606, 0.1203]\", \"lift_point\": \"[0.1606, 0.1203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4868, 0.873]\", \"lift_point\": \"[0.4868, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.3059]\", \"lift_point\": \"[0.0657, 0.3059]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"restaurant in Atlanta\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1606, 0.1203]\", \"lift_point\": \"[0.1606, 0.1203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4868, 0.873]\", \"lift_point\": \"[0.4868, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.3059]\", \"lift_point\": \"[0.0657, 0.3059]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"restaurant in Atlanta\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1991, 0.2296]\", \"lift_point\": \"[0.1991, 0.2296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1606, 0.1203]\", \"lift_point\": \"[0.1606, 0.1203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4868, 0.873]\", \"lift_point\": \"[0.4868, 0.873]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.3059]\", \"lift_point\": \"[0.0657, 0.3059]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"restaurant in Atlanta\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1991, 0.2296]\", \"lift_point\": \"[0.1991, 0.2296]\", \"typed_text\": \"\" Goal: What's a good restaurant in Atlanta?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50752_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\" Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.7958]\", \"lift_point\": \"[0.0589, 0.7958]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.7958]\", \"lift_point\": \"[0.0589, 0.7958]\", \"typed_text\": \"\" Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9347]\", \"lift_point\": \"[0.0649, 0.9347]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.7958]\", \"lift_point\": \"[0.0589, 0.7958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9347]\", \"lift_point\": \"[0.0649, 0.9347]\", \"typed_text\": \"\" Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.7958]\", \"lift_point\": \"[0.0589, 0.7958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9347]\", \"lift_point\": \"[0.0649, 0.9347]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook\" Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.2776]\", \"lift_point\": \"[0.1309, 0.2776]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.7958]\", \"lift_point\": \"[0.0589, 0.7958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9347]\", \"lift_point\": \"[0.0649, 0.9347]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.2776]\", \"lift_point\": \"[0.1309, 0.2776]\", \"typed_text\": \"\" Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1536, 0.8556]\", \"lift_point\": \"[0.1536, 0.8556]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.7958]\", \"lift_point\": \"[0.0589, 0.7958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9347]\", \"lift_point\": \"[0.0649, 0.9347]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.2776]\", \"lift_point\": \"[0.1309, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1536, 0.8556]\", \"lift_point\": \"[0.1536, 0.8556]\", \"typed_text\": \"\" Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1536, 0.854]\", \"lift_point\": \"[0.1536, 0.854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7755, 0.5113]\", \"lift_point\": \"[0.7755, 0.5113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.7958]\", \"lift_point\": \"[0.0589, 0.7958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9347]\", \"lift_point\": \"[0.0649, 0.9347]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"facebook\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1309, 0.2776]\", \"lift_point\": \"[0.1309, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1536, 0.8556]\", \"lift_point\": \"[0.1536, 0.8556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1536, 0.854]\", \"lift_point\": \"[0.1536, 0.854]\", \"typed_text\": \"\" Goal: Check the settings for the Facebook app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50802_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.503]\", \"lift_point\": \"[0.1333, 0.503]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.503]\", \"lift_point\": \"[0.1333, 0.503]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.503]\", \"lift_point\": \"[0.1333, 0.503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.503]\", \"lift_point\": \"[0.1333, 0.503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.7008]\", \"lift_point\": \"[0.7804, 0.7008]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.503]\", \"lift_point\": \"[0.1333, 0.503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7781, 0.7176]\", \"lift_point\": \"[0.7781, 0.7176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.503]\", \"lift_point\": \"[0.1333, 0.503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.4543]\", \"lift_point\": \"[0.658, 0.4543]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.3686]\", \"lift_point\": \"[0.0557, 0.3686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Chicago to London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1333, 0.503]\", \"lift_point\": \"[0.1333, 0.503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.4543]\", \"lift_point\": \"[0.658, 0.4543]\", \"typed_text\": \"\" Goal: Search for flights from Chicago to London>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50888_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50900_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6639]\", \"lift_point\": \"[0.778, 0.6639]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50900_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6639]\", \"lift_point\": \"[0.778, 0.6639]\", \"typed_text\": \"\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.5895]\", \"lift_point\": \"[0.0695, 0.5895]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50900_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6639]\", \"lift_point\": \"[0.778, 0.6639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.5895]\", \"lift_point\": \"[0.0695, 0.5895]\", \"typed_text\": \"\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bangladesh\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50900_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6639]\", \"lift_point\": \"[0.778, 0.6639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.5895]\", \"lift_point\": \"[0.0695, 0.5895]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bangladesh\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.2819]\", \"lift_point\": \"[0.1351, 0.2819]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50900_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6639]\", \"lift_point\": \"[0.778, 0.6639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.5895]\", \"lift_point\": \"[0.0695, 0.5895]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bangladesh\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.2819]\", \"lift_point\": \"[0.1351, 0.2819]\", \"typed_text\": \"\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2566, 0.1763]\", \"lift_point\": \"[0.2566, 0.1763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50900_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.6639]\", \"lift_point\": \"[0.778, 0.6639]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.5895]\", \"lift_point\": \"[0.0695, 0.5895]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Bangladesh\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1351, 0.2819]\", \"lift_point\": \"[0.1351, 0.2819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2566, 0.1763]\", \"lift_point\": \"[0.2566, 0.1763]\", \"typed_text\": \"\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50900_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.67]\", \"lift_point\": \"[0.7736, 0.67]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50935_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.67]\", \"lift_point\": \"[0.7736, 0.67]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0659, 0.9182]\", \"lift_point\": \"[0.0659, 0.9182]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50935_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.67]\", \"lift_point\": \"[0.7736, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0659, 0.9182]\", \"lift_point\": \"[0.0659, 0.9182]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1832, 0.711]\", \"lift_point\": \"[0.1832, 0.711]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50935_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.67]\", \"lift_point\": \"[0.7736, 0.67]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0659, 0.9182]\", \"lift_point\": \"[0.0659, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1832, 0.711]\", \"lift_point\": \"[0.1832, 0.711]\", \"typed_text\": \"\" Goal: Open a new incognito window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50935_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6982]\", \"lift_point\": \"[0.7715, 0.6982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50952_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6982]\", \"lift_point\": \"[0.7715, 0.6982]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5389]\", \"lift_point\": \"[0.0606, 0.5389]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50952_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6982]\", \"lift_point\": \"[0.7715, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5389]\", \"lift_point\": \"[0.0606, 0.5389]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"space technology\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50952_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6982]\", \"lift_point\": \"[0.7715, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5389]\", \"lift_point\": \"[0.0606, 0.5389]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"space technology\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1305, 0.3664]\", \"lift_point\": \"[0.1305, 0.3664]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50952_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6982]\", \"lift_point\": \"[0.7715, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5389]\", \"lift_point\": \"[0.0606, 0.5389]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"space technology\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1305, 0.3664]\", \"lift_point\": \"[0.1305, 0.3664]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2536, 0.1856]\", \"lift_point\": \"[0.2536, 0.1856]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50952_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6982]\", \"lift_point\": \"[0.7715, 0.6982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.5389]\", \"lift_point\": \"[0.0606, 0.5389]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"space technology\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1305, 0.3664]\", \"lift_point\": \"[0.1305, 0.3664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2536, 0.1856]\", \"lift_point\": \"[0.2536, 0.1856]\", \"typed_text\": \"\" Goal: What's the latest news in space technology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50952_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.2215]\", \"lift_point\": \"[0.1255, 0.2215]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.2215]\", \"lift_point\": \"[0.1255, 0.2215]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.2215]\", \"lift_point\": \"[0.1255, 0.2215]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.2215]\", \"lift_point\": \"[0.1255, 0.2215]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6884]\", \"lift_point\": \"[0.7849, 0.6884]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.2215]\", \"lift_point\": \"[0.1255, 0.2215]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.733, 0.472]\", \"lift_point\": \"[0.733, 0.472]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.6896]\", \"lift_point\": \"[0.7824, 0.6896]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0527, 0.471]\", \"lift_point\": \"[0.0527, 0.471]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.2215]\", \"lift_point\": \"[0.1255, 0.2215]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.733, 0.472]\", \"lift_point\": \"[0.733, 0.472]\", \"typed_text\": \"\" Goal: Search for hotels in Denver>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/50970_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.2773]\", \"lift_point\": \"[0.1298, 0.2773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.2773]\", \"lift_point\": \"[0.1298, 0.2773]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3601, 0.3378]\", \"lift_point\": \"[0.3601, 0.3378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.2773]\", \"lift_point\": \"[0.1298, 0.2773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3601, 0.3378]\", \"lift_point\": \"[0.3601, 0.3378]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.2773]\", \"lift_point\": \"[0.1298, 0.2773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3601, 0.3378]\", \"lift_point\": \"[0.3601, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.2773]\", \"lift_point\": \"[0.1298, 0.2773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3601, 0.3378]\", \"lift_point\": \"[0.3601, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6808]\", \"lift_point\": \"[0.7775, 0.6808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.2773]\", \"lift_point\": \"[0.1298, 0.2773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3601, 0.3378]\", \"lift_point\": \"[0.3601, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4589, 0.2733]\", \"lift_point\": \"[0.4589, 0.2733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.4147]\", \"lift_point\": \"[0.0674, 0.4147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in London\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.2773]\", \"lift_point\": \"[0.1298, 0.2773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3601, 0.3378]\", \"lift_point\": \"[0.3601, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4589, 0.2733]\", \"lift_point\": \"[0.4589, 0.2733]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51156_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0704, 0.6117]\", \"lift_point\": \"[0.0704, 0.6117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0704, 0.6117]\", \"lift_point\": \"[0.0704, 0.6117]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.4192]\", \"lift_point\": \"[0.0552, 0.4192]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0704, 0.6117]\", \"lift_point\": \"[0.0704, 0.6117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.4192]\", \"lift_point\": \"[0.0552, 0.4192]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0704, 0.6117]\", \"lift_point\": \"[0.0704, 0.6117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.4192]\", \"lift_point\": \"[0.0552, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0704, 0.6117]\", \"lift_point\": \"[0.0704, 0.6117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.4192]\", \"lift_point\": \"[0.0552, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1496, 0.4698]\", \"lift_point\": \"[0.1496, 0.4698]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0704, 0.6117]\", \"lift_point\": \"[0.0704, 0.6117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.4192]\", \"lift_point\": \"[0.0552, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1496, 0.4698]\", \"lift_point\": \"[0.1496, 0.4698]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5413, 0.2827]\", \"lift_point\": \"[0.5413, 0.2827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7731, 0.6802]\", \"lift_point\": \"[0.7731, 0.6802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0704, 0.6117]\", \"lift_point\": \"[0.0704, 0.6117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.4192]\", \"lift_point\": \"[0.0552, 0.4192]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in New York?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1496, 0.4698]\", \"lift_point\": \"[0.1496, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5413, 0.2827]\", \"lift_point\": \"[0.5413, 0.2827]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51199_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6645]\", \"lift_point\": \"[0.7854, 0.6645]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51417_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6645]\", \"lift_point\": \"[0.7854, 0.6645]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9275]\", \"lift_point\": \"[0.0649, 0.9275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51417_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6645]\", \"lift_point\": \"[0.7854, 0.6645]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9275]\", \"lift_point\": \"[0.0649, 0.9275]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2044, 0.5231]\", \"lift_point\": \"[0.2044, 0.5231]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51417_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7854, 0.6645]\", \"lift_point\": \"[0.7854, 0.6645]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9275]\", \"lift_point\": \"[0.0649, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2044, 0.5231]\", \"lift_point\": \"[0.2044, 0.5231]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51417_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.76, 0.6695]\", \"lift_point\": \"[0.76, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.4571]\", \"lift_point\": \"[0.0646, 0.4571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1339, 0.7196]\", \"lift_point\": \"[0.1339, 0.7196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3983, 0.4939]\", \"lift_point\": \"[0.3983, 0.4939]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7977, 0.8485]\", \"lift_point\": \"[0.7977, 0.8485]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9298, 0.4856]\", \"lift_point\": \"[0.9298, 0.4856]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9298, 0.4856]\", \"lift_point\": \"[0.9298, 0.4856]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.4179]\", \"lift_point\": \"[0.9165, 0.4179]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9298, 0.4856]\", \"lift_point\": \"[0.9298, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3033, 0.6148]\", \"lift_point\": \"[0.3033, 0.6148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9298, 0.4856]\", \"lift_point\": \"[0.9298, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3033, 0.6148]\", \"lift_point\": \"[0.3033, 0.6148]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7561, 0.9427]\", \"lift_point\": \"[0.7561, 0.9427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9298, 0.4856]\", \"lift_point\": \"[0.9298, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3033, 0.6148]\", \"lift_point\": \"[0.3033, 0.6148]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.898, 0.4587]\", \"lift_point\": \"[0.898, 0.4587]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9303, 0.4941]\", \"lift_point\": \"[0.9303, 0.4941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9298, 0.4856]\", \"lift_point\": \"[0.9298, 0.4856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3033, 0.6148]\", \"lift_point\": \"[0.3033, 0.6148]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.898, 0.4587]\", \"lift_point\": \"[0.898, 0.4587]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51504_24.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.7128]\", \"lift_point\": \"[0.776, 0.7128]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51533_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.7128]\", \"lift_point\": \"[0.776, 0.7128]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5586]\", \"lift_point\": \"[0.0651, 0.5586]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51533_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.7128]\", \"lift_point\": \"[0.776, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5586]\", \"lift_point\": \"[0.0651, 0.5586]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in san diego\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51533_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.7128]\", \"lift_point\": \"[0.776, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5586]\", \"lift_point\": \"[0.0651, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in san diego\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1422, 0.3437]\", \"lift_point\": \"[0.1422, 0.3437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51533_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.7128]\", \"lift_point\": \"[0.776, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5586]\", \"lift_point\": \"[0.0651, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1422, 0.3437]\", \"lift_point\": \"[0.1422, 0.3437]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.4097]\", \"lift_point\": \"[0.3651, 0.4097]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51533_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.7128]\", \"lift_point\": \"[0.776, 0.7128]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5586]\", \"lift_point\": \"[0.0651, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in san diego\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1422, 0.3437]\", \"lift_point\": \"[0.1422, 0.3437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3651, 0.4097]\", \"lift_point\": \"[0.3651, 0.4097]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51533_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.6251]\", \"lift_point\": \"[0.882, 0.6251]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51662_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.6251]\", \"lift_point\": \"[0.882, 0.6251]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"mascara\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51662_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.6251]\", \"lift_point\": \"[0.882, 0.6251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"mascara\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2502, 0.316]\", \"lift_point\": \"[0.2502, 0.316]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51662_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.6251]\", \"lift_point\": \"[0.882, 0.6251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"mascara\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2502, 0.316]\", \"lift_point\": \"[0.2502, 0.316]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2995, 0.2829]\", \"lift_point\": \"[0.2995, 0.2829]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51662_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.6251]\", \"lift_point\": \"[0.882, 0.6251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"mascara\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2502, 0.316]\", \"lift_point\": \"[0.2502, 0.316]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2995, 0.2829]\", \"lift_point\": \"[0.2995, 0.2829]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51662_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51687_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.893, 0.3801]\", \"lift_point\": \"[0.893, 0.3801]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51687_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.893, 0.3801]\", \"lift_point\": \"[0.893, 0.3801]\", \"typed_text\": \"\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good chinese restaurant \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51687_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.893, 0.3801]\", \"lift_point\": \"[0.893, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good chinese restaurant \" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.6063]\", \"lift_point\": \"[0.1215, 0.6063]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51687_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.893, 0.3801]\", \"lift_point\": \"[0.893, 0.3801]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good chinese restaurant \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.6063]\", \"lift_point\": \"[0.1215, 0.6063]\", \"typed_text\": \"\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51687_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Sydney to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7764, 0.6817]\", \"lift_point\": \"[0.7764, 0.6817]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51693_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7764, 0.6817]\", \"lift_point\": \"[0.7764, 0.6817]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.3885]\", \"lift_point\": \"[0.0628, 0.3885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51693_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7764, 0.6817]\", \"lift_point\": \"[0.7764, 0.6817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.3885]\", \"lift_point\": \"[0.0628, 0.3885]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Helsinki>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51693_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7764, 0.6817]\", \"lift_point\": \"[0.7764, 0.6817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.3885]\", \"lift_point\": \"[0.0628, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Helsinki\" Goal: Search for flights from Sydney to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.4238]\", \"lift_point\": \"[0.1365, 0.4238]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51693_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7764, 0.6817]\", \"lift_point\": \"[0.7764, 0.6817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.3885]\", \"lift_point\": \"[0.0628, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.4238]\", \"lift_point\": \"[0.1365, 0.4238]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3855, 0.2661]\", \"lift_point\": \"[0.3855, 0.2661]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51693_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7764, 0.6817]\", \"lift_point\": \"[0.7764, 0.6817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0628, 0.3885]\", \"lift_point\": \"[0.0628, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Sydney to Helsinki\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.4238]\", \"lift_point\": \"[0.1365, 0.4238]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3855, 0.2661]\", \"lift_point\": \"[0.3855, 0.2661]\", \"typed_text\": \"\" Goal: Search for flights from Sydney to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/51693_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8029, 0.6321]\", \"lift_point\": \"[0.8029, 0.6321]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52056_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8029, 0.6321]\", \"lift_point\": \"[0.8029, 0.6321]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.4386]\", \"lift_point\": \"[0.05, 0.4386]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52056_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8029, 0.6321]\", \"lift_point\": \"[0.8029, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.4386]\", \"lift_point\": \"[0.05, 0.4386]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"speed of a jet\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52056_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8029, 0.6321]\", \"lift_point\": \"[0.8029, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.4386]\", \"lift_point\": \"[0.05, 0.4386]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"speed of a jet\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.4935]\", \"lift_point\": \"[0.1347, 0.4935]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52056_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8029, 0.6321]\", \"lift_point\": \"[0.8029, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.05, 0.4386]\", \"lift_point\": \"[0.05, 0.4386]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"speed of a jet\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.4935]\", \"lift_point\": \"[0.1347, 0.4935]\", \"typed_text\": \"\" Goal: What is the speed of a jet?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52056_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52096_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3924, 0.663]\", \"lift_point\": \"[0.3924, 0.663]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52096_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3924, 0.663]\", \"lift_point\": \"[0.3924, 0.663]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52096_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.962]\", \"lift_point\": \"[0.1823, 0.962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.962]\", \"lift_point\": \"[0.1823, 0.962]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.962]\", \"lift_point\": \"[0.1823, 0.962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.3319]\", \"lift_point\": \"[0.7884, 0.3319]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.962]\", \"lift_point\": \"[0.1823, 0.962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.3319]\", \"lift_point\": \"[0.7884, 0.3319]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0476, 0.3254]\", \"lift_point\": \"[0.0476, 0.3254]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.962]\", \"lift_point\": \"[0.1823, 0.962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.3319]\", \"lift_point\": \"[0.7884, 0.3319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0476, 0.3254]\", \"lift_point\": \"[0.0476, 0.3254]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google Play Movies app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.962]\", \"lift_point\": \"[0.1823, 0.962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.3319]\", \"lift_point\": \"[0.7884, 0.3319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0476, 0.3254]\", \"lift_point\": \"[0.0476, 0.3254]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google Play Movies app\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1178, 0.4862]\", \"lift_point\": \"[0.1178, 0.4862]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.4203]\", \"lift_point\": \"[0.7679, 0.4203]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1823, 0.962]\", \"lift_point\": \"[0.1823, 0.962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.3319]\", \"lift_point\": \"[0.7884, 0.3319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0476, 0.3254]\", \"lift_point\": \"[0.0476, 0.3254]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google Play Movies app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1178, 0.4862]\", \"lift_point\": \"[0.1178, 0.4862]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Movies app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52200_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.15, 0.2057]\", \"lift_point\": \"[0.15, 0.2057]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7935]\", \"lift_point\": \"[0.0615, 0.7935]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1057, 0.6469]\", \"lift_point\": \"[0.1057, 0.6469]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.7715]\", \"lift_point\": \"[0.0564, 0.7715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.8126]\", \"lift_point\": \"[0.0639, 0.8126]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2059, 0.2981]\", \"lift_point\": \"[0.2059, 0.2981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.8126]\", \"lift_point\": \"[0.0639, 0.8126]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.9422]\", \"lift_point\": \"[0.059, 0.9422]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.8126]\", \"lift_point\": \"[0.0639, 0.8126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.9422]\", \"lift_point\": \"[0.059, 0.9422]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twittwer\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.8126]\", \"lift_point\": \"[0.0639, 0.8126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.9422]\", \"lift_point\": \"[0.059, 0.9422]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twittwer\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1684, 0.3521]\", \"lift_point\": \"[0.1684, 0.3521]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.8126]\", \"lift_point\": \"[0.0639, 0.8126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.9422]\", \"lift_point\": \"[0.059, 0.9422]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twittwer\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1684, 0.3521]\", \"lift_point\": \"[0.1684, 0.3521]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1151, 0.4543]\", \"lift_point\": \"[0.1151, 0.4543]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.539, 0.4292]\", \"lift_point\": \"[0.539, 0.4292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0515, 0.0421]\", \"lift_point\": \"[0.0515, 0.0421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.8126]\", \"lift_point\": \"[0.0639, 0.8126]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.9422]\", \"lift_point\": \"[0.059, 0.9422]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twittwer\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1684, 0.3521]\", \"lift_point\": \"[0.1684, 0.3521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1151, 0.4543]\", \"lift_point\": \"[0.1151, 0.4543]\", \"typed_text\": \"\" Goal: Check the settings for the Twitter app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52290_20.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6538]\", \"lift_point\": \"[0.779, 0.6538]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52582_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6538]\", \"lift_point\": \"[0.779, 0.6538]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0489, 0.528]\", \"lift_point\": \"[0.0489, 0.528]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52582_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6538]\", \"lift_point\": \"[0.779, 0.6538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0489, 0.528]\", \"lift_point\": \"[0.0489, 0.528]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Burger King?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52582_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6538]\", \"lift_point\": \"[0.779, 0.6538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0489, 0.528]\", \"lift_point\": \"[0.0489, 0.528]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Burger King?\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.3364]\", \"lift_point\": \"[0.1172, 0.3364]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52582_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.6538]\", \"lift_point\": \"[0.779, 0.6538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0489, 0.528]\", \"lift_point\": \"[0.0489, 0.528]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Burger King?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.3364]\", \"lift_point\": \"[0.1172, 0.3364]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52582_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest tech news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6613]\", \"lift_point\": \"[0.8006, 0.6613]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52611_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6613]\", \"lift_point\": \"[0.8006, 0.6613]\", \"typed_text\": \"\" Goal: What's the latest tech news?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.4418]\", \"lift_point\": \"[0.052, 0.4418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52611_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6613]\", \"lift_point\": \"[0.8006, 0.6613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.4418]\", \"lift_point\": \"[0.052, 0.4418]\", \"typed_text\": \"\" Goal: What's the latest tech news?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest tech news?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52611_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6613]\", \"lift_point\": \"[0.8006, 0.6613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.4418]\", \"lift_point\": \"[0.052, 0.4418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest tech news?\" Goal: What's the latest tech news?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52611_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6613]\", \"lift_point\": \"[0.8006, 0.6613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.4418]\", \"lift_point\": \"[0.052, 0.4418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest tech news?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest tech news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1186, 0.2666]\", \"lift_point\": \"[0.1186, 0.2666]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52611_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6613]\", \"lift_point\": \"[0.8006, 0.6613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.4418]\", \"lift_point\": \"[0.052, 0.4418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest tech news?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1186, 0.2666]\", \"lift_point\": \"[0.1186, 0.2666]\", \"typed_text\": \"\" Goal: What's the latest tech news?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2403, 0.1912]\", \"lift_point\": \"[0.2403, 0.1912]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52611_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6613]\", \"lift_point\": \"[0.8006, 0.6613]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.4418]\", \"lift_point\": \"[0.052, 0.4418]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest tech news?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1186, 0.2666]\", \"lift_point\": \"[0.1186, 0.2666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2403, 0.1912]\", \"lift_point\": \"[0.2403, 0.1912]\", \"typed_text\": \"\" Goal: What's the latest tech news?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52611_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [STATUS_TASK_IMPOSSIBLE]\n; Action Decision: \"action_type\": \"STATUS_TASK_IMPOSSIBLE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52663_1.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8973, 0.4526]\", \"lift_point\": \"[0.8973, 0.4526]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52726_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8973, 0.4526]\", \"lift_point\": \"[0.8973, 0.4526]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5975, 0.4096]\", \"lift_point\": \"[0.5975, 0.4096]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52726_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8973, 0.4526]\", \"lift_point\": \"[0.8973, 0.4526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5975, 0.4096]\", \"lift_point\": \"[0.5975, 0.4096]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.5058]\", \"lift_point\": \"[0.4163, 0.5058]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52726_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8973, 0.4526]\", \"lift_point\": \"[0.8973, 0.4526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5975, 0.4096]\", \"lift_point\": \"[0.5975, 0.4096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.5058]\", \"lift_point\": \"[0.4163, 0.5058]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3482, 0.7223]\", \"lift_point\": \"[0.3482, 0.7223]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52726_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8973, 0.4526]\", \"lift_point\": \"[0.8973, 0.4526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5975, 0.4096]\", \"lift_point\": \"[0.5975, 0.4096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.5058]\", \"lift_point\": \"[0.4163, 0.5058]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3482, 0.7223]\", \"lift_point\": \"[0.3482, 0.7223]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6747, 0.7048]\", \"lift_point\": \"[0.6747, 0.7048]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52726_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8973, 0.4526]\", \"lift_point\": \"[0.8973, 0.4526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5975, 0.4096]\", \"lift_point\": \"[0.5975, 0.4096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4163, 0.5058]\", \"lift_point\": \"[0.4163, 0.5058]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3482, 0.7223]\", \"lift_point\": \"[0.3482, 0.7223]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6747, 0.7048]\", \"lift_point\": \"[0.6747, 0.7048]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52726_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from San Francisco to Tokyo>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52737_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from San Francisco to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7881, 0.5788]\", \"lift_point\": \"[0.7881, 0.5788]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52737_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7881, 0.5788]\", \"lift_point\": \"[0.7881, 0.5788]\", \"typed_text\": \"\" Goal: Search for flights from San Francisco to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0484, 0.4007]\", \"lift_point\": \"[0.0484, 0.4007]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52737_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7881, 0.5788]\", \"lift_point\": \"[0.7881, 0.5788]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0484, 0.4007]\", \"lift_point\": \"[0.0484, 0.4007]\", \"typed_text\": \"\" Goal: Search for flights from San Francisco to Tokyo>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Francisco to Tokyo\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52737_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7881, 0.5788]\", \"lift_point\": \"[0.7881, 0.5788]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0484, 0.4007]\", \"lift_point\": \"[0.0484, 0.4007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Francisco to Tokyo\" Goal: Search for flights from San Francisco to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1087, 0.2467]\", \"lift_point\": \"[0.1087, 0.2467]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52737_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7881, 0.5788]\", \"lift_point\": \"[0.7881, 0.5788]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0484, 0.4007]\", \"lift_point\": \"[0.0484, 0.4007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Francisco to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1087, 0.2467]\", \"lift_point\": \"[0.1087, 0.2467]\", \"typed_text\": \"\" Goal: Search for flights from San Francisco to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2371, 0.1914]\", \"lift_point\": \"[0.2371, 0.1914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52737_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7881, 0.5788]\", \"lift_point\": \"[0.7881, 0.5788]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0484, 0.4007]\", \"lift_point\": \"[0.0484, 0.4007]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from San Francisco to Tokyo\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1087, 0.2467]\", \"lift_point\": \"[0.1087, 0.2467]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2371, 0.1914]\", \"lift_point\": \"[0.2371, 0.1914]\", \"typed_text\": \"\" Goal: Search for flights from San Francisco to Tokyo>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52737_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52751_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.5856]\", \"lift_point\": \"[0.7938, 0.5856]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52751_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.5856]\", \"lift_point\": \"[0.7938, 0.5856]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.3825]\", \"lift_point\": \"[0.0542, 0.3825]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52751_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.5856]\", \"lift_point\": \"[0.7938, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.3825]\", \"lift_point\": \"[0.0542, 0.3825]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in Zurich\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52751_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.5856]\", \"lift_point\": \"[0.7938, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.3825]\", \"lift_point\": \"[0.0542, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in Zurich\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52751_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.5856]\", \"lift_point\": \"[0.7938, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.3825]\", \"lift_point\": \"[0.0542, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in Zurich\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.2172]\", \"lift_point\": \"[0.1172, 0.2172]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52751_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.5856]\", \"lift_point\": \"[0.7938, 0.5856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0542, 0.3825]\", \"lift_point\": \"[0.0542, 0.3825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" hotels in Zurich\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1172, 0.2172]\", \"lift_point\": \"[0.1172, 0.2172]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/52751_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.2902]\", \"lift_point\": \"[0.1201, 0.2902]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.2902]\", \"lift_point\": \"[0.1201, 0.2902]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.2902]\", \"lift_point\": \"[0.1201, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5219, 0.5729]\", \"lift_point\": \"[0.5219, 0.5729]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.2902]\", \"lift_point\": \"[0.1201, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5219, 0.5729]\", \"lift_point\": \"[0.5219, 0.5729]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7719, 0.8908]\", \"lift_point\": \"[0.7719, 0.8908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.2902]\", \"lift_point\": \"[0.1201, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5219, 0.5729]\", \"lift_point\": \"[0.5219, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4217, 0.584]\", \"lift_point\": \"[0.4217, 0.584]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0418, 0.8196]\", \"lift_point\": \"[0.0418, 0.8196]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3589]\", \"lift_point\": \"[0.0583, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"GameSpot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.2902]\", \"lift_point\": \"[0.1201, 0.2902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5219, 0.5729]\", \"lift_point\": \"[0.5219, 0.5729]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4217, 0.584]\", \"lift_point\": \"[0.4217, 0.584]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot News?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53261_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5385, 0.7858]\", \"lift_point\": \"[0.5385, 0.7858]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8897, 0.2232]\", \"lift_point\": \"[0.8897, 0.2232]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather like in Seoul\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.79, 0.4597]\", \"lift_point\": \"[0.79, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6875, 0.5881]\", \"lift_point\": \"[0.6875, 0.5881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.792, 0.6095]\", \"lift_point\": \"[0.792, 0.6095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6065, 0.3781]\", \"lift_point\": \"[0.6065, 0.3781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6102, 0.6719]\", \"lift_point\": \"[0.6102, 0.6719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6118, 0.6305]\", \"lift_point\": \"[0.6118, 0.6305]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7374, 0.9374]\", \"lift_point\": \"[0.7374, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7359, 0.9374]\", \"lift_point\": \"[0.7359, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7345, 0.9374]\", \"lift_point\": \"[0.7345, 0.9374]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7406, 0.9516]\", \"lift_point\": \"[0.7406, 0.9516]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_54.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_55.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_56.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_57.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_58.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_59.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_60.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_61.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_62.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_63.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5144, 0.9441]\", \"lift_point\": \"[0.5144, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_64.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.563, 0.9304]\", \"lift_point\": \"[0.563, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_65.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_66.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5609, 0.9437]\", \"lift_point\": \"[0.5609, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_67.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5624, 0.9437]\", \"lift_point\": \"[0.5624, 0.9437]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_68.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_69.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_70.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_71.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_72.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_73.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_74.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_75.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_76.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5594, 0.9406]\", \"lift_point\": \"[0.5594, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_77.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_78.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_79.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_80.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_81.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_82.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_83.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_84.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_85.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_86.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_87.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_88.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_89.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_90.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_91.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_92.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_93.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_94.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_95.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_96.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_97.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_98.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_99.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_100.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_101.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.558, 0.9406]\", \"lift_point\": \"[0.558, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_102.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_103.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_104.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_105.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_106.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_107.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_108.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_109.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_110.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_111.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_112.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_113.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_114.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_115.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_116.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_117.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_118.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_119.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_120.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_121.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_122.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_123.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_124.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_125.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_126.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_127.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_128.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_129.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_130.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_131.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5565, 0.9406]\", \"lift_point\": \"[0.5565, 0.9406]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.9377]\", \"lift_point\": \"[0.555, 0.9377]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53409_132.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.7143]\", \"lift_point\": \"[0.7663, 0.7143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.7143]\", \"lift_point\": \"[0.7663, 0.7143]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.7043]\", \"lift_point\": \"[0.7736, 0.7043]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.7143]\", \"lift_point\": \"[0.7663, 0.7143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.7043]\", \"lift_point\": \"[0.7736, 0.7043]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5959]\", \"lift_point\": \"[0.0534, 0.5959]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.7143]\", \"lift_point\": \"[0.7663, 0.7143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.7043]\", \"lift_point\": \"[0.7736, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5959]\", \"lift_point\": \"[0.0534, 0.5959]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Nordstrom\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.7143]\", \"lift_point\": \"[0.7663, 0.7143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.7043]\", \"lift_point\": \"[0.7736, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5959]\", \"lift_point\": \"[0.0534, 0.5959]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Nordstrom\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7722, 0.6917]\", \"lift_point\": \"[0.7722, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7831, 0.6797]\", \"lift_point\": \"[0.7831, 0.6797]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.7143]\", \"lift_point\": \"[0.7663, 0.7143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7736, 0.7043]\", \"lift_point\": \"[0.7736, 0.7043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.5959]\", \"lift_point\": \"[0.0534, 0.5959]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Nordstrom\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Nordstrom?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53494_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53689_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3412, 0.8923]\", \"lift_point\": \"[0.3412, 0.8923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53689_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3412, 0.8923]\", \"lift_point\": \"[0.3412, 0.8923]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.8823]\", \"lift_point\": \"[0.3547, 0.8823]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53689_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3412, 0.8923]\", \"lift_point\": \"[0.3412, 0.8923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3547, 0.8823]\", \"lift_point\": \"[0.3547, 0.8823]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53689_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.7007]\", \"lift_point\": \"[0.7777, 0.7007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.6695]\", \"lift_point\": \"[0.7765, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7687, 0.7081]\", \"lift_point\": \"[0.7687, 0.7081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.77, 0.6836]\", \"lift_point\": \"[0.77, 0.6836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5575]\", \"lift_point\": \"[0.0578, 0.5575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest video from the Wall Street Journal\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.4459]\", \"lift_point\": \"[0.1119, 0.4459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.4129]\", \"lift_point\": \"[0.1197, 0.4129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2193, 0.1494]\", \"lift_point\": \"[0.2193, 0.1494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.3515]\", \"lift_point\": \"[0.4137, 0.3515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2259, 0.1519]\", \"lift_point\": \"[0.2259, 0.1519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.3515]\", \"lift_point\": \"[0.4137, 0.3515]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.948]\", \"lift_point\": \"[0.336, 0.948]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2295, 0.1764]\", \"lift_point\": \"[0.2295, 0.1764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.3515]\", \"lift_point\": \"[0.4137, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.948]\", \"lift_point\": \"[0.336, 0.948]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2488, 0.8637]\", \"lift_point\": \"[0.2488, 0.8637]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4161, 0.1821]\", \"lift_point\": \"[0.4161, 0.1821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.3515]\", \"lift_point\": \"[0.4137, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.948]\", \"lift_point\": \"[0.336, 0.948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2488, 0.8637]\", \"lift_point\": \"[0.2488, 0.8637]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.4777]\", \"lift_point\": \"[0.1115, 0.4777]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4099, 0.9441]\", \"lift_point\": \"[0.4099, 0.9441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.3515]\", \"lift_point\": \"[0.4137, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.948]\", \"lift_point\": \"[0.336, 0.948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2488, 0.8637]\", \"lift_point\": \"[0.2488, 0.8637]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.4777]\", \"lift_point\": \"[0.1115, 0.4777]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1713, 0.5039]\", \"lift_point\": \"[0.1713, 0.5039]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3227, 0.1137]\", \"lift_point\": \"[0.3227, 0.1137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8646, 0.8766]\", \"lift_point\": \"[0.8646, 0.8766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4137, 0.3515]\", \"lift_point\": \"[0.4137, 0.3515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.336, 0.948]\", \"lift_point\": \"[0.336, 0.948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2488, 0.8637]\", \"lift_point\": \"[0.2488, 0.8637]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.4777]\", \"lift_point\": \"[0.1115, 0.4777]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1713, 0.5039]\", \"lift_point\": \"[0.1713, 0.5039]\", \"typed_text\": \"\" Goal: Play the latest video from the Wall Street Journal>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53762_22.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.7074]\", \"lift_point\": \"[0.7791, 0.7074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.7074]\", \"lift_point\": \"[0.7791, 0.7074]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.7322]\", \"lift_point\": \"[0.7692, 0.7322]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.7074]\", \"lift_point\": \"[0.7791, 0.7074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.7322]\", \"lift_point\": \"[0.7692, 0.7322]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.664]\", \"lift_point\": \"[0.0565, 0.664]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.7074]\", \"lift_point\": \"[0.7791, 0.7074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.7322]\", \"lift_point\": \"[0.7692, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.664]\", \"lift_point\": \"[0.0565, 0.664]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.7074]\", \"lift_point\": \"[0.7791, 0.7074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.7322]\", \"lift_point\": \"[0.7692, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.664]\", \"lift_point\": \"[0.0565, 0.664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Galaxy phone\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.7074]\", \"lift_point\": \"[0.7791, 0.7074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.7322]\", \"lift_point\": \"[0.7692, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.664]\", \"lift_point\": \"[0.0565, 0.664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Galaxy phone\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.2212]\", \"lift_point\": \"[0.1119, 0.2212]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7742, 0.6796]\", \"lift_point\": \"[0.7742, 0.6796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7791, 0.7074]\", \"lift_point\": \"[0.7791, 0.7074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7692, 0.7322]\", \"lift_point\": \"[0.7692, 0.7322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.664]\", \"lift_point\": \"[0.0565, 0.664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Galaxy phone\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1119, 0.2212]\", \"lift_point\": \"[0.1119, 0.2212]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/53955_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Zurich to Helsinki\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Zurich to Helsinki\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Zurich to Helsinki\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Zurich to Helsinki\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4804, 0.5038]\", \"lift_point\": \"[0.4804, 0.5038]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Zurich to Helsinki\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4804, 0.5038]\", \"lift_point\": \"[0.4804, 0.5038]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4934]\", \"lift_point\": \"[0.48, 0.4934]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Zurich to Helsinki\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4804, 0.5038]\", \"lift_point\": \"[0.4804, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4934]\", \"lift_point\": \"[0.48, 0.4934]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4789, 0.602]\", \"lift_point\": \"[0.4789, 0.602]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7653, 0.683]\", \"lift_point\": \"[0.7653, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.6573]\", \"lift_point\": \"[0.0632, 0.6573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Zurich to Helsinki\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4804, 0.5038]\", \"lift_point\": \"[0.4804, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4934]\", \"lift_point\": \"[0.48, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4789, 0.602]\", \"lift_point\": \"[0.4789, 0.602]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Helsinki>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54066_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.6959]\", \"lift_point\": \"[0.7758, 0.6959]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54138_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.6959]\", \"lift_point\": \"[0.7758, 0.6959]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6775]\", \"lift_point\": \"[0.7825, 0.6775]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54138_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.6959]\", \"lift_point\": \"[0.7758, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6775]\", \"lift_point\": \"[0.7825, 0.6775]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.765, 0.7127]\", \"lift_point\": \"[0.765, 0.7127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54138_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.6959]\", \"lift_point\": \"[0.7758, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6775]\", \"lift_point\": \"[0.7825, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.765, 0.7127]\", \"lift_point\": \"[0.765, 0.7127]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.6044]\", \"lift_point\": \"[0.059, 0.6044]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54138_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.6959]\", \"lift_point\": \"[0.7758, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6775]\", \"lift_point\": \"[0.7825, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.765, 0.7127]\", \"lift_point\": \"[0.765, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.6044]\", \"lift_point\": \"[0.059, 0.6044]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Venezuela\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54138_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.6959]\", \"lift_point\": \"[0.7758, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6775]\", \"lift_point\": \"[0.7825, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.765, 0.7127]\", \"lift_point\": \"[0.765, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.6044]\", \"lift_point\": \"[0.059, 0.6044]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Venezuela\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54138_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.6959]\", \"lift_point\": \"[0.7758, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6775]\", \"lift_point\": \"[0.7825, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.765, 0.7127]\", \"lift_point\": \"[0.765, 0.7127]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.6044]\", \"lift_point\": \"[0.059, 0.6044]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Venezuela\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54138_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5435]\", \"lift_point\": \"[0.0578, 0.5435]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5435]\", \"lift_point\": \"[0.0578, 0.5435]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5435]\", \"lift_point\": \"[0.0578, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5435]\", \"lift_point\": \"[0.0578, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5435]\", \"lift_point\": \"[0.0578, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5435]\", \"lift_point\": \"[0.0578, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1866, 0.5036]\", \"lift_point\": \"[0.1866, 0.5036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7698, 0.6702]\", \"lift_point\": \"[0.7698, 0.6702]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.5435]\", \"lift_point\": \"[0.0578, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1866, 0.5036]\", \"lift_point\": \"[0.1866, 0.5036]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54204_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the speed of light?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.6854]\", \"lift_point\": \"[0.7777, 0.6854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.6854]\", \"lift_point\": \"[0.7777, 0.6854]\", \"typed_text\": \"\" Goal: What's the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0523, 0.7214]\", \"lift_point\": \"[0.0523, 0.7214]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.6854]\", \"lift_point\": \"[0.7777, 0.6854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0523, 0.7214]\", \"lift_point\": \"[0.0523, 0.7214]\", \"typed_text\": \"\" Goal: What's the speed of light?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the speed of light\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.6854]\", \"lift_point\": \"[0.7777, 0.6854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0523, 0.7214]\", \"lift_point\": \"[0.0523, 0.7214]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the speed of light\" Goal: What's the speed of light?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.6854]\", \"lift_point\": \"[0.7777, 0.6854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0523, 0.7214]\", \"lift_point\": \"[0.0523, 0.7214]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the speed of light\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.6854]\", \"lift_point\": \"[0.7777, 0.6854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0523, 0.7214]\", \"lift_point\": \"[0.0523, 0.7214]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the speed of light\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the speed of light?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8326, 0.2399]\", \"lift_point\": \"[0.8326, 0.2399]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7777, 0.6854]\", \"lift_point\": \"[0.7777, 0.6854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0523, 0.7214]\", \"lift_point\": \"[0.0523, 0.7214]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the speed of light\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8326, 0.2399]\", \"lift_point\": \"[0.8326, 0.2399]\", \"typed_text\": \"\" Goal: What's the speed of light?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54281_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54355_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3462, 0.1159]\", \"lift_point\": \"[0.3462, 0.1159]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54355_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3462, 0.1159]\", \"lift_point\": \"[0.3462, 0.1159]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.1268]\", \"lift_point\": \"[0.3493, 0.1268]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54355_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3462, 0.1159]\", \"lift_point\": \"[0.3462, 0.1159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.1268]\", \"lift_point\": \"[0.3493, 0.1268]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9081, 0.9309]\", \"lift_point\": \"[0.9081, 0.9309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54355_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3462, 0.1159]\", \"lift_point\": \"[0.3462, 0.1159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.1268]\", \"lift_point\": \"[0.3493, 0.1268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9081, 0.9309]\", \"lift_point\": \"[0.9081, 0.9309]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8776, 0.5248]\", \"lift_point\": \"[0.8776, 0.5248]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54355_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3462, 0.1159]\", \"lift_point\": \"[0.3462, 0.1159]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3493, 0.1268]\", \"lift_point\": \"[0.3493, 0.1268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9081, 0.9309]\", \"lift_point\": \"[0.9081, 0.9309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8776, 0.5248]\", \"lift_point\": \"[0.8776, 0.5248]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54355_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6216]\", \"lift_point\": \"[0.0551, 0.6216]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6216]\", \"lift_point\": \"[0.0551, 0.6216]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Tokyo to NYC\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6216]\", \"lift_point\": \"[0.0551, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Tokyo to NYC\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6216]\", \"lift_point\": \"[0.0551, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Tokyo to NYC\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6216]\", \"lift_point\": \"[0.0551, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Tokyo to NYC\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6339, 0.5404]\", \"lift_point\": \"[0.6339, 0.5404]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.768, 0.6793]\", \"lift_point\": \"[0.768, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7656, 0.6793]\", \"lift_point\": \"[0.7656, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6216]\", \"lift_point\": \"[0.0551, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Tokyo to NYC\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6339, 0.5404]\", \"lift_point\": \"[0.6339, 0.5404]\", \"typed_text\": \"\" Goal: Search for flights from Tokyo to NYC>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54378_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54494_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54494_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54494_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6035]\", \"lift_point\": \"[0.0551, 0.6035]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54494_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6035]\", \"lift_point\": \"[0.0551, 0.6035]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Mexico\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54494_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6035]\", \"lift_point\": \"[0.0551, 0.6035]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Mexico\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54494_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7718, 0.6811]\", \"lift_point\": \"[0.7718, 0.6811]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.6035]\", \"lift_point\": \"[0.0551, 0.6035]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Mexico\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54494_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54513_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3544, 0.8683]\", \"lift_point\": \"[0.3544, 0.8683]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54513_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3544, 0.8683]\", \"lift_point\": \"[0.3544, 0.8683]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54513_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.6201]\", \"lift_point\": \"[0.0601, 0.6201]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.6201]\", \"lift_point\": \"[0.0601, 0.6201]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in New York\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.6201]\", \"lift_point\": \"[0.0601, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in New York\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.6201]\", \"lift_point\": \"[0.0601, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in New York\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.6201]\", \"lift_point\": \"[0.0601, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in New York\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.6201]\", \"lift_point\": \"[0.0601, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in New York\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5394, 0.5373]\", \"lift_point\": \"[0.5394, 0.5373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7753, 0.6942]\", \"lift_point\": \"[0.7753, 0.6942]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.6201]\", \"lift_point\": \"[0.0601, 0.6201]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in New York\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5394, 0.5373]\", \"lift_point\": \"[0.5394, 0.5373]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54550_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest Burger King\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest Burger King\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest Burger King\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest Burger King\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest Burger King\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.3592]\", \"lift_point\": \"[0.3498, 0.3592]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7641, 0.6697]\", \"lift_point\": \"[0.7641, 0.6697]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0495, 0.6581]\", \"lift_point\": \"[0.0495, 0.6581]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest Burger King\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.3592]\", \"lift_point\": \"[0.3498, 0.3592]\", \"typed_text\": \"\" Goal: How do I get to the nearest Burger King?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54560_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7769, 0.6849]\", \"lift_point\": \"[0.7769, 0.6849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7769, 0.6849]\", \"lift_point\": \"[0.7769, 0.6849]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.6473]\", \"lift_point\": \"[0.0557, 0.6473]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7769, 0.6849]\", \"lift_point\": \"[0.7769, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.6473]\", \"lift_point\": \"[0.0557, 0.6473]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Boston\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7769, 0.6849]\", \"lift_point\": \"[0.7769, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.6473]\", \"lift_point\": \"[0.0557, 0.6473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Boston\" Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7769, 0.6849]\", \"lift_point\": \"[0.7769, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.6473]\", \"lift_point\": \"[0.0557, 0.6473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Boston\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7769, 0.6849]\", \"lift_point\": \"[0.7769, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.6473]\", \"lift_point\": \"[0.0557, 0.6473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Boston\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3434, 0.5535]\", \"lift_point\": \"[0.3434, 0.5535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7679, 0.6849]\", \"lift_point\": \"[0.7679, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7769, 0.6849]\", \"lift_point\": \"[0.7769, 0.6849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0557, 0.6473]\", \"lift_point\": \"[0.0557, 0.6473]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Mexico city to Boston\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3434, 0.5535]\", \"lift_point\": \"[0.3434, 0.5535]\", \"typed_text\": \"\" Goal: Search for flights from Mexico city to Boston>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54579_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54596_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54596_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4652, 0.4809]\", \"lift_point\": \"[0.4652, 0.4809]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54596_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4652, 0.4809]\", \"lift_point\": \"[0.4652, 0.4809]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.2105]\", \"lift_point\": \"[0.0504, 0.2105]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54596_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4652, 0.4809]\", \"lift_point\": \"[0.4652, 0.4809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.2105]\", \"lift_point\": \"[0.0504, 0.2105]\", \"typed_text\": \"\" Goal: Open the downloads>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54596_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1495, 0.1703]\", \"lift_point\": \"[0.1495, 0.1703]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1495, 0.1703]\", \"lift_point\": \"[0.1495, 0.1703]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1088, 0.0608]\", \"lift_point\": \"[0.1088, 0.0608]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6905]\", \"lift_point\": \"[0.774, 0.6905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1495, 0.1703]\", \"lift_point\": \"[0.1495, 0.1703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1088, 0.0608]\", \"lift_point\": \"[0.1088, 0.0608]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1916, 0.1381]\", \"lift_point\": \"[0.1916, 0.1381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.6655]\", \"lift_point\": \"[0.0548, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Brazil\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1495, 0.1703]\", \"lift_point\": \"[0.1495, 0.1703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1088, 0.0608]\", \"lift_point\": \"[0.1088, 0.0608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1916, 0.1381]\", \"lift_point\": \"[0.1916, 0.1381]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54618_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9022, 0.4373]\", \"lift_point\": \"[0.9022, 0.4373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54633_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9022, 0.4373]\", \"lift_point\": \"[0.9022, 0.4373]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach tent\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54633_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9022, 0.4373]\", \"lift_point\": \"[0.9022, 0.4373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach tent\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1303, 0.3898]\", \"lift_point\": \"[0.1303, 0.3898]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54633_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9022, 0.4373]\", \"lift_point\": \"[0.9022, 0.4373]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"where can i buy a nice beach tent\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1303, 0.3898]\", \"lift_point\": \"[0.1303, 0.3898]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach tent?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54633_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my gmail>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5113, 0.4446]\", \"lift_point\": \"[0.5113, 0.4446]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5113, 0.4446]\", \"lift_point\": \"[0.5113, 0.4446]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5113, 0.4446]\", \"lift_point\": \"[0.5113, 0.4446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5113, 0.4446]\", \"lift_point\": \"[0.5113, 0.4446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4532, 0.8557]\", \"lift_point\": \"[0.4532, 0.8557]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5113, 0.4446]\", \"lift_point\": \"[0.5113, 0.4446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4532, 0.8557]\", \"lift_point\": \"[0.4532, 0.8557]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5238, 0.8381]\", \"lift_point\": \"[0.5238, 0.8381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5105, 0.493]\", \"lift_point\": \"[0.5105, 0.493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5113, 0.4446]\", \"lift_point\": \"[0.5113, 0.4446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9118, 0.4912]\", \"lift_point\": \"[0.9118, 0.4912]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4532, 0.8557]\", \"lift_point\": \"[0.4532, 0.8557]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5238, 0.8381]\", \"lift_point\": \"[0.5238, 0.8381]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54772_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.093, 0.1289]\", \"lift_point\": \"[0.093, 0.1289]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.093, 0.1289]\", \"lift_point\": \"[0.093, 0.1289]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.4967]\", \"lift_point\": \"[0.8862, 0.4967]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.093, 0.1289]\", \"lift_point\": \"[0.093, 0.1289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.4967]\", \"lift_point\": \"[0.8862, 0.4967]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4615, 0.319]\", \"lift_point\": \"[0.4615, 0.319]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.093, 0.1289]\", \"lift_point\": \"[0.093, 0.1289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.4967]\", \"lift_point\": \"[0.8862, 0.4967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4615, 0.319]\", \"lift_point\": \"[0.4615, 0.319]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4212, 0.5109]\", \"lift_point\": \"[0.4212, 0.5109]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.093, 0.1289]\", \"lift_point\": \"[0.093, 0.1289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.4967]\", \"lift_point\": \"[0.8862, 0.4967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4615, 0.319]\", \"lift_point\": \"[0.4615, 0.319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4212, 0.5109]\", \"lift_point\": \"[0.4212, 0.5109]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2988, 0.7573]\", \"lift_point\": \"[0.2988, 0.7573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.093, 0.1289]\", \"lift_point\": \"[0.093, 0.1289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.4967]\", \"lift_point\": \"[0.8862, 0.4967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4615, 0.319]\", \"lift_point\": \"[0.4615, 0.319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4212, 0.5109]\", \"lift_point\": \"[0.4212, 0.5109]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2988, 0.7573]\", \"lift_point\": \"[0.2988, 0.7573]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6884, 0.7371]\", \"lift_point\": \"[0.6884, 0.7371]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3586, 0.7019]\", \"lift_point\": \"[0.3586, 0.7019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.093, 0.1289]\", \"lift_point\": \"[0.093, 0.1289]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.4967]\", \"lift_point\": \"[0.8862, 0.4967]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4615, 0.319]\", \"lift_point\": \"[0.4615, 0.319]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4212, 0.5109]\", \"lift_point\": \"[0.4212, 0.5109]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2988, 0.7573]\", \"lift_point\": \"[0.2988, 0.7573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6884, 0.7371]\", \"lift_point\": \"[0.6884, 0.7371]\", \"typed_text\": \"\" Goal: Set an alarm for 10am>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54820_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.7155]\", \"lift_point\": \"[0.7757, 0.7155]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.7155]\", \"lift_point\": \"[0.7757, 0.7155]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.3216]\", \"lift_point\": \"[0.0565, 0.3216]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.7155]\", \"lift_point\": \"[0.7757, 0.7155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.3216]\", \"lift_point\": \"[0.0565, 0.3216]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of switzerland\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.7155]\", \"lift_point\": \"[0.7757, 0.7155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.3216]\", \"lift_point\": \"[0.0565, 0.3216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of switzerland\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.2655]\", \"lift_point\": \"[0.1283, 0.2655]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.7155]\", \"lift_point\": \"[0.7757, 0.7155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.3216]\", \"lift_point\": \"[0.0565, 0.3216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of switzerland\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.2655]\", \"lift_point\": \"[0.1283, 0.2655]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5061, 0.0977]\", \"lift_point\": \"[0.5061, 0.0977]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.781, 0.6938]\", \"lift_point\": \"[0.781, 0.6938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5761, 0.9414]\", \"lift_point\": \"[0.5761, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.7155]\", \"lift_point\": \"[0.7757, 0.7155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.3216]\", \"lift_point\": \"[0.0565, 0.3216]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of switzerland\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1283, 0.2655]\", \"lift_point\": \"[0.1283, 0.2655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5061, 0.0977]\", \"lift_point\": \"[0.5061, 0.0977]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54851_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is the universe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6867]\", \"lift_point\": \"[0.7891, 0.6867]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54891_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6867]\", \"lift_point\": \"[0.7891, 0.6867]\", \"typed_text\": \"\" Goal: How big is the universe?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3792]\", \"lift_point\": \"[0.0583, 0.3792]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54891_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6867]\", \"lift_point\": \"[0.7891, 0.6867]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3792]\", \"lift_point\": \"[0.0583, 0.3792]\", \"typed_text\": \"\" Goal: How big is the universe?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the universe\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54891_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6867]\", \"lift_point\": \"[0.7891, 0.6867]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3792]\", \"lift_point\": \"[0.0583, 0.3792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the universe\" Goal: How big is the universe?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.2762]\", \"lift_point\": \"[0.1308, 0.2762]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54891_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7891, 0.6867]\", \"lift_point\": \"[0.7891, 0.6867]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0583, 0.3792]\", \"lift_point\": \"[0.0583, 0.3792]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is the universe\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.2762]\", \"lift_point\": \"[0.1308, 0.2762]\", \"typed_text\": \"\" Goal: How big is the universe?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54891_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.5041]\", \"lift_point\": \"[0.7818, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3813, 0.459]\", \"lift_point\": \"[0.3813, 0.459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0389]\", \"lift_point\": \"[0.0728, 0.0389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3813, 0.459]\", \"lift_point\": \"[0.3813, 0.459]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.9391]\", \"lift_point\": \"[0.3313, 0.9391]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0513, 0.8113]\", \"lift_point\": \"[0.0513, 0.8113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3813, 0.459]\", \"lift_point\": \"[0.3813, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.9391]\", \"lift_point\": \"[0.3313, 0.9391]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1975, 0.5157]\", \"lift_point\": \"[0.1975, 0.5157]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0548, 0.3905]\", \"lift_point\": \"[0.0548, 0.3905]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3813, 0.459]\", \"lift_point\": \"[0.3813, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.9391]\", \"lift_point\": \"[0.3313, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1975, 0.5157]\", \"lift_point\": \"[0.1975, 0.5157]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1608, 0.4889]\", \"lift_point\": \"[0.1608, 0.4889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" latest video from the Huffington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3813, 0.459]\", \"lift_point\": \"[0.3813, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.9391]\", \"lift_point\": \"[0.3313, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1975, 0.5157]\", \"lift_point\": \"[0.1975, 0.5157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1608, 0.4889]\", \"lift_point\": \"[0.1608, 0.4889]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.4998]\", \"lift_point\": \"[0.1787, 0.4998]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1108, 0.3338]\", \"lift_point\": \"[0.1108, 0.3338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3813, 0.459]\", \"lift_point\": \"[0.3813, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.9391]\", \"lift_point\": \"[0.3313, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1975, 0.5157]\", \"lift_point\": \"[0.1975, 0.5157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1608, 0.4889]\", \"lift_point\": \"[0.1608, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.4998]\", \"lift_point\": \"[0.1787, 0.4998]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.4998]\", \"lift_point\": \"[0.1787, 0.4998]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3813, 0.459]\", \"lift_point\": \"[0.3813, 0.459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3313, 0.9391]\", \"lift_point\": \"[0.3313, 0.9391]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1975, 0.5157]\", \"lift_point\": \"[0.1975, 0.5157]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1608, 0.4889]\", \"lift_point\": \"[0.1608, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.4998]\", \"lift_point\": \"[0.1787, 0.4998]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.4998]\", \"lift_point\": \"[0.1787, 0.4998]\", \"typed_text\": \"\" Goal: Play the latest video from the Huffington Post>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54929_16.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7816, 0.7066]\", \"lift_point\": \"[0.7816, 0.7066]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54935_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7816, 0.7066]\", \"lift_point\": \"[0.7816, 0.7066]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4272]\", \"lift_point\": \"[0.0606, 0.4272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54935_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7816, 0.7066]\", \"lift_point\": \"[0.7816, 0.7066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4272]\", \"lift_point\": \"[0.0606, 0.4272]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Las Vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54935_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7816, 0.7066]\", \"lift_point\": \"[0.7816, 0.7066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4272]\", \"lift_point\": \"[0.0606, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Las Vegas\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1295, 0.2443]\", \"lift_point\": \"[0.1295, 0.2443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54935_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7816, 0.7066]\", \"lift_point\": \"[0.7816, 0.7066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4272]\", \"lift_point\": \"[0.0606, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1295, 0.2443]\", \"lift_point\": \"[0.1295, 0.2443]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4195, 0.2697]\", \"lift_point\": \"[0.4195, 0.2697]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54935_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7816, 0.7066]\", \"lift_point\": \"[0.7816, 0.7066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4272]\", \"lift_point\": \"[0.0606, 0.4272]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1295, 0.2443]\", \"lift_point\": \"[0.1295, 0.2443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4195, 0.2697]\", \"lift_point\": \"[0.4195, 0.2697]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54935_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6856]\", \"lift_point\": \"[0.791, 0.6856]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54948_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6856]\", \"lift_point\": \"[0.791, 0.6856]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.3815]\", \"lift_point\": \"[0.0608, 0.3815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54948_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6856]\", \"lift_point\": \"[0.791, 0.6856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.3815]\", \"lift_point\": \"[0.0608, 0.3815]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Puerto Rico news\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54948_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6856]\", \"lift_point\": \"[0.791, 0.6856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.3815]\", \"lift_point\": \"[0.0608, 0.3815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Puerto Rico news\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.125, 0.2552]\", \"lift_point\": \"[0.125, 0.2552]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54948_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6856]\", \"lift_point\": \"[0.791, 0.6856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.3815]\", \"lift_point\": \"[0.0608, 0.3815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Puerto Rico news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.125, 0.2552]\", \"lift_point\": \"[0.125, 0.2552]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2505, 0.1546]\", \"lift_point\": \"[0.2505, 0.1546]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54948_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6856]\", \"lift_point\": \"[0.791, 0.6856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.3815]\", \"lift_point\": \"[0.0608, 0.3815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Puerto Rico news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.125, 0.2552]\", \"lift_point\": \"[0.125, 0.2552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2505, 0.1546]\", \"lift_point\": \"[0.2505, 0.1546]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5026, 0.2729]\", \"lift_point\": \"[0.5026, 0.2729]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54948_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6856]\", \"lift_point\": \"[0.791, 0.6856]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.3815]\", \"lift_point\": \"[0.0608, 0.3815]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Puerto Rico news\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.125, 0.2552]\", \"lift_point\": \"[0.125, 0.2552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2505, 0.1546]\", \"lift_point\": \"[0.2505, 0.1546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5026, 0.2729]\", \"lift_point\": \"[0.5026, 0.2729]\", \"typed_text\": \"\" Goal: What's the news in Puerto Rico?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/54948_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news this week?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.694]\", \"lift_point\": \"[0.7788, 0.694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.694]\", \"lift_point\": \"[0.7788, 0.694]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7658, 0.6924]\", \"lift_point\": \"[0.7658, 0.6924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.694]\", \"lift_point\": \"[0.7788, 0.694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7658, 0.6924]\", \"lift_point\": \"[0.7658, 0.6924]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.4574]\", \"lift_point\": \"[0.0673, 0.4574]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.694]\", \"lift_point\": \"[0.7788, 0.694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7658, 0.6924]\", \"lift_point\": \"[0.7658, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.4574]\", \"lift_point\": \"[0.0673, 0.4574]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.694]\", \"lift_point\": \"[0.7788, 0.694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7658, 0.6924]\", \"lift_point\": \"[0.7658, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.4574]\", \"lift_point\": \"[0.0673, 0.4574]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.1713]\", \"lift_point\": \"[0.1219, 0.1713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.694]\", \"lift_point\": \"[0.7788, 0.694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7658, 0.6924]\", \"lift_point\": \"[0.7658, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.4574]\", \"lift_point\": \"[0.0673, 0.4574]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.1713]\", \"lift_point\": \"[0.1219, 0.1713]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.2272]\", \"lift_point\": \"[0.4387, 0.2272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7788, 0.694]\", \"lift_point\": \"[0.7788, 0.694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7658, 0.6924]\", \"lift_point\": \"[0.7658, 0.6924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.4574]\", \"lift_point\": \"[0.0673, 0.4574]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1219, 0.1713]\", \"lift_point\": \"[0.1219, 0.1713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.2272]\", \"lift_point\": \"[0.4387, 0.2272]\", \"typed_text\": \"\" Goal: What's the news this week?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55137_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55160_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6931]\", \"lift_point\": \"[0.7756, 0.6931]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55160_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6931]\", \"lift_point\": \"[0.7756, 0.6931]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.2776]\", \"lift_point\": \"[0.0603, 0.2776]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55160_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6931]\", \"lift_point\": \"[0.7756, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.2776]\", \"lift_point\": \"[0.0603, 0.2776]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Walmart\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55160_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6931]\", \"lift_point\": \"[0.7756, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.2776]\", \"lift_point\": \"[0.0603, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Walmart\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1268, 0.1721]\", \"lift_point\": \"[0.1268, 0.1721]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55160_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7756, 0.6931]\", \"lift_point\": \"[0.7756, 0.6931]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.2776]\", \"lift_point\": \"[0.0603, 0.2776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1268, 0.1721]\", \"lift_point\": \"[0.1268, 0.1721]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55160_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Vizio TV\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Vizio TV\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.1186]\", \"lift_point\": \"[0.0653, 0.1186]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Vizio TV\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.1186]\", \"lift_point\": \"[0.0653, 0.1186]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.5659]\", \"lift_point\": \"[0.1762, 0.5659]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Vizio TV\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.1186]\", \"lift_point\": \"[0.0653, 0.1186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.5659]\", \"lift_point\": \"[0.1762, 0.5659]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0707, 0.5071]\", \"lift_point\": \"[0.0707, 0.5071]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Vizio TV\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.1186]\", \"lift_point\": \"[0.0653, 0.1186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.5659]\", \"lift_point\": \"[0.1762, 0.5659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0707, 0.5071]\", \"lift_point\": \"[0.0707, 0.5071]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1253, 0.2932]\", \"lift_point\": \"[0.1253, 0.2932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Vizio TV\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.1186]\", \"lift_point\": \"[0.0653, 0.1186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.5659]\", \"lift_point\": \"[0.1762, 0.5659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0707, 0.5071]\", \"lift_point\": \"[0.0707, 0.5071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1253, 0.2932]\", \"lift_point\": \"[0.1253, 0.2932]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2479, 0.2464]\", \"lift_point\": \"[0.2479, 0.2464]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7861, 0.6766]\", \"lift_point\": \"[0.7861, 0.6766]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4081]\", \"lift_point\": \"[0.0576, 0.4081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the Vizio TV\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.1186]\", \"lift_point\": \"[0.0653, 0.1186]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1762, 0.5659]\", \"lift_point\": \"[0.1762, 0.5659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0707, 0.5071]\", \"lift_point\": \"[0.0707, 0.5071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1253, 0.2932]\", \"lift_point\": \"[0.1253, 0.2932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2479, 0.2464]\", \"lift_point\": \"[0.2479, 0.2464]\", \"typed_text\": \"\" Goal: What's the price of the Vizio TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55525_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6895]\", \"lift_point\": \"[0.7792, 0.6895]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55571_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6895]\", \"lift_point\": \"[0.7792, 0.6895]\", \"typed_text\": \"\" Goal: What time is it in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.4226]\", \"lift_point\": \"[0.0559, 0.4226]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55571_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6895]\", \"lift_point\": \"[0.7792, 0.6895]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.4226]\", \"lift_point\": \"[0.0559, 0.4226]\", \"typed_text\": \"\" Goal: What time is it in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in san francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55571_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6895]\", \"lift_point\": \"[0.7792, 0.6895]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.4226]\", \"lift_point\": \"[0.0559, 0.4226]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in san francisco\" Goal: What time is it in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2649]\", \"lift_point\": \"[0.1263, 0.2649]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55571_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7792, 0.6895]\", \"lift_point\": \"[0.7792, 0.6895]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0559, 0.4226]\", \"lift_point\": \"[0.0559, 0.4226]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in san francisco\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1263, 0.2649]\", \"lift_point\": \"[0.1263, 0.2649]\", \"typed_text\": \"\" Goal: What time is it in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55571_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4273, 0.3883]\", \"lift_point\": \"[0.4273, 0.3883]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4273, 0.3883]\", \"lift_point\": \"[0.4273, 0.3883]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.5981]\", \"lift_point\": \"[0.8495, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4273, 0.3883]\", \"lift_point\": \"[0.4273, 0.3883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4273, 0.3883]\", \"lift_point\": \"[0.4273, 0.3883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3126, 0.7238]\", \"lift_point\": \"[0.3126, 0.7238]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1492, 0.2963]\", \"lift_point\": \"[0.1492, 0.2963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4273, 0.3883]\", \"lift_point\": \"[0.4273, 0.3883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3126, 0.7238]\", \"lift_point\": \"[0.3126, 0.7238]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3169, 0.7431]\", \"lift_point\": \"[0.3169, 0.7431]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0728, 0.0778]\", \"lift_point\": \"[0.0728, 0.0778]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4273, 0.3883]\", \"lift_point\": \"[0.4273, 0.3883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3126, 0.7238]\", \"lift_point\": \"[0.3126, 0.7238]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3169, 0.7431]\", \"lift_point\": \"[0.3169, 0.7431]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.7381]\", \"lift_point\": \"[0.6816, 0.7381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0718, 0.1017]\", \"lift_point\": \"[0.0718, 0.1017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9291, 0.5142]\", \"lift_point\": \"[0.9291, 0.5142]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4273, 0.3883]\", \"lift_point\": \"[0.4273, 0.3883]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4219, 0.5065]\", \"lift_point\": \"[0.4219, 0.5065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3126, 0.7238]\", \"lift_point\": \"[0.3126, 0.7238]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3169, 0.7431]\", \"lift_point\": \"[0.3169, 0.7431]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.7381]\", \"lift_point\": \"[0.6816, 0.7381]\", \"typed_text\": \"\" Goal: Set an alarm for 11am>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55653_16.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55674_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55674_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1757, 0.8474]\", \"lift_point\": \"[0.1757, 0.8474]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55674_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1757, 0.8474]\", \"lift_point\": \"[0.1757, 0.8474]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0754, 0.5816]\", \"lift_point\": \"[0.0754, 0.5816]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55674_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1757, 0.8474]\", \"lift_point\": \"[0.1757, 0.8474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0754, 0.5816]\", \"lift_point\": \"[0.0754, 0.5816]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" top rated burger restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55674_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1757, 0.8474]\", \"lift_point\": \"[0.1757, 0.8474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0754, 0.5816]\", \"lift_point\": \"[0.0754, 0.5816]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" top rated burger restaurants\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1531, 0.5397]\", \"lift_point\": \"[0.1531, 0.5397]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55674_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1757, 0.8474]\", \"lift_point\": \"[0.1757, 0.8474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0754, 0.5816]\", \"lift_point\": \"[0.0754, 0.5816]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" top rated burger restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1531, 0.5397]\", \"lift_point\": \"[0.1531, 0.5397]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55674_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55745_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3812]\", \"lift_point\": \"[0.8843, 0.3812]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55745_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3812]\", \"lift_point\": \"[0.8843, 0.3812]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55745_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3812]\", \"lift_point\": \"[0.8843, 0.3812]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.7572]\", \"lift_point\": \"[0.1379, 0.7572]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55745_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3812]\", \"lift_point\": \"[0.8843, 0.3812]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a rocket?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.7572]\", \"lift_point\": \"[0.1379, 0.7572]\", \"typed_text\": \"\" Goal: What is the speed of a rocket?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55745_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8882, 0.4486]\", \"lift_point\": \"[0.8882, 0.4486]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Miami\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3618, 0.5251]\", \"lift_point\": \"[0.3618, 0.5251]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.3545]\", \"lift_point\": \"[0.1329, 0.3545]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3618, 0.5251]\", \"lift_point\": \"[0.3618, 0.5251]\", \"typed_text\": \"\" Goal: Search for hotels in Miami>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55758_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.3466]\", \"lift_point\": \"[0.2559, 0.3466]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.3466]\", \"lift_point\": \"[0.2559, 0.3466]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest JCPenney\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.3466]\", \"lift_point\": \"[0.2559, 0.3466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest JCPenney\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.3638]\", \"lift_point\": \"[0.1115, 0.3638]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.3466]\", \"lift_point\": \"[0.2559, 0.3466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest JCPenney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.3638]\", \"lift_point\": \"[0.1115, 0.3638]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1095, 0.535]\", \"lift_point\": \"[0.1095, 0.535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.6852]\", \"lift_point\": \"[0.7914, 0.6852]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6827]\", \"lift_point\": \"[0.7988, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.3466]\", \"lift_point\": \"[0.2559, 0.3466]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest JCPenney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1115, 0.3638]\", \"lift_point\": \"[0.1115, 0.3638]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1095, 0.535]\", \"lift_point\": \"[0.1095, 0.535]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55811_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55918_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55918_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.498]\", \"lift_point\": \"[0.0553, 0.498]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55918_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.498]\", \"lift_point\": \"[0.0553, 0.498]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0522, 0.9411]\", \"lift_point\": \"[0.0522, 0.9411]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55918_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.498]\", \"lift_point\": \"[0.0553, 0.498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0522, 0.9411]\", \"lift_point\": \"[0.0522, 0.9411]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55918_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.498]\", \"lift_point\": \"[0.0553, 0.498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0522, 0.9411]\", \"lift_point\": \"[0.0522, 0.9411]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.3262]\", \"lift_point\": \"[0.116, 0.3262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55918_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6695]\", \"lift_point\": \"[0.7979, 0.6695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.498]\", \"lift_point\": \"[0.0553, 0.498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0522, 0.9411]\", \"lift_point\": \"[0.0522, 0.9411]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation (skincare) product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.3262]\", \"lift_point\": \"[0.116, 0.3262]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55918_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55938_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55938_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.6959]\", \"lift_point\": \"[0.7868, 0.6959]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55938_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.6959]\", \"lift_point\": \"[0.7868, 0.6959]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.6959]\", \"lift_point\": \"[0.7868, 0.6959]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55938_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.6959]\", \"lift_point\": \"[0.7868, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7868, 0.6959]\", \"lift_point\": \"[0.7868, 0.6959]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/55938_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.6352]\", \"lift_point\": \"[0.047, 0.6352]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.6352]\", \"lift_point\": \"[0.047, 0.6352]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.9329]\", \"lift_point\": \"[0.0472, 0.9329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.6352]\", \"lift_point\": \"[0.047, 0.6352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.9329]\", \"lift_point\": \"[0.0472, 0.9329]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.6352]\", \"lift_point\": \"[0.047, 0.6352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.9329]\", \"lift_point\": \"[0.0472, 0.9329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1136, 0.2422]\", \"lift_point\": \"[0.1136, 0.2422]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.6352]\", \"lift_point\": \"[0.047, 0.6352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.9329]\", \"lift_point\": \"[0.0472, 0.9329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1136, 0.2422]\", \"lift_point\": \"[0.1136, 0.2422]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.6352]\", \"lift_point\": \"[0.047, 0.6352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.9329]\", \"lift_point\": \"[0.0472, 0.9329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1136, 0.2422]\", \"lift_point\": \"[0.1136, 0.2422]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3436, 0.5238]\", \"lift_point\": \"[0.3436, 0.5238]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7948, 0.6923]\", \"lift_point\": \"[0.7948, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.047, 0.6352]\", \"lift_point\": \"[0.047, 0.6352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0472, 0.9329]\", \"lift_point\": \"[0.0472, 0.9329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best Mexican restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1136, 0.2422]\", \"lift_point\": \"[0.1136, 0.2422]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3436, 0.5238]\", \"lift_point\": \"[0.3436, 0.5238]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56230_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6927]\", \"lift_point\": \"[0.7849, 0.6927]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6927]\", \"lift_point\": \"[0.7849, 0.6927]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4434]\", \"lift_point\": \"[0.0562, 0.4434]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6927]\", \"lift_point\": \"[0.7849, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4434]\", \"lift_point\": \"[0.0562, 0.4434]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9411]\", \"lift_point\": \"[0.0461, 0.9411]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6927]\", \"lift_point\": \"[0.7849, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4434]\", \"lift_point\": \"[0.0562, 0.4434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9411]\", \"lift_point\": \"[0.0461, 0.9411]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Venezuela\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6927]\", \"lift_point\": \"[0.7849, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4434]\", \"lift_point\": \"[0.0562, 0.4434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9411]\", \"lift_point\": \"[0.0461, 0.9411]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Venezuela\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6927]\", \"lift_point\": \"[0.7849, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4434]\", \"lift_point\": \"[0.0562, 0.4434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9411]\", \"lift_point\": \"[0.0461, 0.9411]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Venezuela\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2158, 0.1338]\", \"lift_point\": \"[0.2158, 0.1338]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7886, 0.6901]\", \"lift_point\": \"[0.7886, 0.6901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7849, 0.6927]\", \"lift_point\": \"[0.7849, 0.6927]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.4434]\", \"lift_point\": \"[0.0562, 0.4434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.9411]\", \"lift_point\": \"[0.0461, 0.9411]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Venezuela\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2158, 0.1338]\", \"lift_point\": \"[0.2158, 0.1338]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56270_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7982, 0.6829]\", \"lift_point\": \"[0.7982, 0.6829]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7982, 0.6829]\", \"lift_point\": \"[0.7982, 0.6829]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.6829]\", \"lift_point\": \"[0.7994, 0.6829]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7982, 0.6829]\", \"lift_point\": \"[0.7982, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.6829]\", \"lift_point\": \"[0.7994, 0.6829]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.5387]\", \"lift_point\": \"[0.0492, 0.5387]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7982, 0.6829]\", \"lift_point\": \"[0.7982, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.6829]\", \"lift_point\": \"[0.7994, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.5387]\", \"lift_point\": \"[0.0492, 0.5387]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.9319]\", \"lift_point\": \"[0.0475, 0.9319]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7982, 0.6829]\", \"lift_point\": \"[0.7982, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.6829]\", \"lift_point\": \"[0.7994, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.5387]\", \"lift_point\": \"[0.0492, 0.5387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.9319]\", \"lift_point\": \"[0.0475, 0.9319]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Colombia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7982, 0.6829]\", \"lift_point\": \"[0.7982, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.6829]\", \"lift_point\": \"[0.7994, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.5387]\", \"lift_point\": \"[0.0492, 0.5387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.9319]\", \"lift_point\": \"[0.0475, 0.9319]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Colombia\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7969, 0.6805]\", \"lift_point\": \"[0.7969, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7982, 0.6829]\", \"lift_point\": \"[0.7982, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.6829]\", \"lift_point\": \"[0.7994, 0.6829]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0492, 0.5387]\", \"lift_point\": \"[0.0492, 0.5387]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.9319]\", \"lift_point\": \"[0.0475, 0.9319]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" capital of Colombia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Colombia>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56351_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56381_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7919, 0.6831]\", \"lift_point\": \"[0.7919, 0.6831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56381_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7919, 0.6831]\", \"lift_point\": \"[0.7919, 0.6831]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.6831]\", \"lift_point\": \"[0.7907, 0.6831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56381_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7919, 0.6831]\", \"lift_point\": \"[0.7919, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.6831]\", \"lift_point\": \"[0.7907, 0.6831]\", \"typed_text\": \"\" Goal: Open a new window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56381_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5798]\", \"lift_point\": \"[0.1198, 0.5798]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5798]\", \"lift_point\": \"[0.1198, 0.5798]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.8973]\", \"lift_point\": \"[0.0635, 0.8973]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5798]\", \"lift_point\": \"[0.1198, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.8973]\", \"lift_point\": \"[0.0635, 0.8973]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.2929]\", \"lift_point\": \"[0.0722, 0.2929]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0, 0.7479]\", \"lift_point\": \"[0.0, 0.7479]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5798]\", \"lift_point\": \"[0.1198, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.8973]\", \"lift_point\": \"[0.0635, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.2929]\", \"lift_point\": \"[0.0722, 0.2929]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in San Francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5798]\", \"lift_point\": \"[0.1198, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.8973]\", \"lift_point\": \"[0.0635, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.2929]\", \"lift_point\": \"[0.0722, 0.2929]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in San Francisco\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.3994]\", \"lift_point\": \"[0.1297, 0.3994]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.509, 0.6815]\", \"lift_point\": \"[0.509, 0.6815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5798]\", \"lift_point\": \"[0.1198, 0.5798]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.8973]\", \"lift_point\": \"[0.0635, 0.8973]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.2929]\", \"lift_point\": \"[0.0722, 0.2929]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in San Francisco\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.3994]\", \"lift_point\": \"[0.1297, 0.3994]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56514_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56565_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9023, 0.3274]\", \"lift_point\": \"[0.9023, 0.3274]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56565_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9023, 0.3274]\", \"lift_point\": \"[0.9023, 0.3274]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.2528]\", \"lift_point\": \"[0.0744, 0.2528]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56565_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9023, 0.3274]\", \"lift_point\": \"[0.9023, 0.3274]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.2528]\", \"lift_point\": \"[0.0744, 0.2528]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56565_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9023, 0.3274]\", \"lift_point\": \"[0.9023, 0.3274]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.2528]\", \"lift_point\": \"[0.0744, 0.2528]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1367, 0.3651]\", \"lift_point\": \"[0.1367, 0.3651]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56565_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9023, 0.3274]\", \"lift_point\": \"[0.9023, 0.3274]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.2528]\", \"lift_point\": \"[0.0744, 0.2528]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1367, 0.3651]\", \"lift_point\": \"[0.1367, 0.3651]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56565_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8772, 0.5148]\", \"lift_point\": \"[0.8772, 0.5148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56655_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8772, 0.5148]\", \"lift_point\": \"[0.8772, 0.5148]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameTrailers?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56655_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8772, 0.5148]\", \"lift_point\": \"[0.8772, 0.5148]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameTrailers?\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.4555]\", \"lift_point\": \"[0.1396, 0.4555]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56655_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8772, 0.5148]\", \"lift_point\": \"[0.8772, 0.5148]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameTrailers?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1396, 0.4555]\", \"lift_point\": \"[0.1396, 0.4555]\", \"typed_text\": \"\" Goal: What's the latest video from GameTrailers?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56655_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8748, 0.3711]\", \"lift_point\": \"[0.8748, 0.3711]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56699_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8748, 0.3711]\", \"lift_point\": \"[0.8748, 0.3711]\", \"typed_text\": \"\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4529, 0.1284]\", \"lift_point\": \"[0.4529, 0.1284]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56699_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8748, 0.3711]\", \"lift_point\": \"[0.8748, 0.3711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4529, 0.1284]\", \"lift_point\": \"[0.4529, 0.1284]\", \"typed_text\": \"\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3438]\", \"lift_point\": \"[0.0655, 0.3438]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56699_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8748, 0.3711]\", \"lift_point\": \"[0.8748, 0.3711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4529, 0.1284]\", \"lift_point\": \"[0.4529, 0.1284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3438]\", \"lift_point\": \"[0.0655, 0.3438]\", \"typed_text\": \"\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Chinese restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56699_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8748, 0.3711]\", \"lift_point\": \"[0.8748, 0.3711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4529, 0.1284]\", \"lift_point\": \"[0.4529, 0.1284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3438]\", \"lift_point\": \"[0.0655, 0.3438]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Chinese restaurants\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1542, 0.5827]\", \"lift_point\": \"[0.1542, 0.5827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56699_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8748, 0.3711]\", \"lift_point\": \"[0.8748, 0.3711]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4529, 0.1284]\", \"lift_point\": \"[0.4529, 0.1284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.3438]\", \"lift_point\": \"[0.0655, 0.3438]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Chinese restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1542, 0.5827]\", \"lift_point\": \"[0.1542, 0.5827]\", \"typed_text\": \"\" Goal: Search for good Chinese restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56699_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3867, 0.7017]\", \"lift_point\": \"[0.3867, 0.7017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3867, 0.7017]\", \"lift_point\": \"[0.3867, 0.7017]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.083]\", \"lift_point\": \"[0.0817, 0.083]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3867, 0.7017]\", \"lift_point\": \"[0.3867, 0.7017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.083]\", \"lift_point\": \"[0.0817, 0.083]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8808, 0.483]\", \"lift_point\": \"[0.8808, 0.483]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3867, 0.7017]\", \"lift_point\": \"[0.3867, 0.7017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.083]\", \"lift_point\": \"[0.0817, 0.083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8808, 0.483]\", \"lift_point\": \"[0.8808, 0.483]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4037, 0.503]\", \"lift_point\": \"[0.4037, 0.503]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3867, 0.7017]\", \"lift_point\": \"[0.3867, 0.7017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.083]\", \"lift_point\": \"[0.0817, 0.083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8808, 0.483]\", \"lift_point\": \"[0.8808, 0.483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4037, 0.503]\", \"lift_point\": \"[0.4037, 0.503]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4139, 0.5104]\", \"lift_point\": \"[0.4139, 0.5104]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8901, 0.5255]\", \"lift_point\": \"[0.8901, 0.5255]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3867, 0.7017]\", \"lift_point\": \"[0.3867, 0.7017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.083]\", \"lift_point\": \"[0.0817, 0.083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8808, 0.483]\", \"lift_point\": \"[0.8808, 0.483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4037, 0.503]\", \"lift_point\": \"[0.4037, 0.503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4139, 0.5104]\", \"lift_point\": \"[0.4139, 0.5104]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7124, 0.7722]\", \"lift_point\": \"[0.7124, 0.7722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3867, 0.7017]\", \"lift_point\": \"[0.3867, 0.7017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.083]\", \"lift_point\": \"[0.0817, 0.083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8808, 0.483]\", \"lift_point\": \"[0.8808, 0.483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4037, 0.503]\", \"lift_point\": \"[0.4037, 0.503]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4139, 0.5104]\", \"lift_point\": \"[0.4139, 0.5104]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7124, 0.7722]\", \"lift_point\": \"[0.7124, 0.7722]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56758_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56826_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.1296]\", \"lift_point\": \"[0.882, 0.1296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56826_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.1296]\", \"lift_point\": \"[0.882, 0.1296]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Austin?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56826_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.1296]\", \"lift_point\": \"[0.882, 0.1296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Austin?\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0536, 0.3399]\", \"lift_point\": \"[0.0536, 0.3399]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56826_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.1296]\", \"lift_point\": \"[0.882, 0.1296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Austin?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0536, 0.3399]\", \"lift_point\": \"[0.0536, 0.3399]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56826_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.882, 0.1296]\", \"lift_point\": \"[0.882, 0.1296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Austin?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0536, 0.3399]\", \"lift_point\": \"[0.0536, 0.3399]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56826_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Venezuela>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56845_1.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Wikipedia app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56851_1.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Apple Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6964]\", \"lift_point\": \"[0.7875, 0.6964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56978_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6964]\", \"lift_point\": \"[0.7875, 0.6964]\", \"typed_text\": \"\" Goal: How do I get to the nearest Apple Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.4258]\", \"lift_point\": \"[0.1715, 0.4258]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56978_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6964]\", \"lift_point\": \"[0.7875, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.4258]\", \"lift_point\": \"[0.1715, 0.4258]\", \"typed_text\": \"\" Goal: How do I get to the nearest Apple Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.9217]\", \"lift_point\": \"[0.1171, 0.9217]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56978_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6964]\", \"lift_point\": \"[0.7875, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.4258]\", \"lift_point\": \"[0.1715, 0.4258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.9217]\", \"lift_point\": \"[0.1171, 0.9217]\", \"typed_text\": \"\" Goal: How do I get to the nearest Apple Store?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Apple Store?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56978_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6964]\", \"lift_point\": \"[0.7875, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.4258]\", \"lift_point\": \"[0.1715, 0.4258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.9217]\", \"lift_point\": \"[0.1171, 0.9217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Apple Store?\" Goal: How do I get to the nearest Apple Store?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56978_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6964]\", \"lift_point\": \"[0.7875, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1715, 0.4258]\", \"lift_point\": \"[0.1715, 0.4258]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.9217]\", \"lift_point\": \"[0.1171, 0.9217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Apple Store?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Apple Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/56978_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57138_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.2804]\", \"lift_point\": \"[0.5618, 0.2804]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57138_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.2804]\", \"lift_point\": \"[0.5618, 0.2804]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8338, 0.5854]\", \"lift_point\": \"[0.8338, 0.5854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57138_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.2804]\", \"lift_point\": \"[0.5618, 0.2804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8338, 0.5854]\", \"lift_point\": \"[0.8338, 0.5854]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.5531]\", \"lift_point\": \"[0.5045, 0.5531]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57138_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5618, 0.2804]\", \"lift_point\": \"[0.5618, 0.2804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8338, 0.5854]\", \"lift_point\": \"[0.8338, 0.5854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5045, 0.5531]\", \"lift_point\": \"[0.5045, 0.5531]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57138_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5938, 0.718]\", \"lift_point\": \"[0.5938, 0.718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1164, 0.1471]\", \"lift_point\": \"[0.1164, 0.1471]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8339, 0.5343]\", \"lift_point\": \"[0.8339, 0.5343]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3829, 0.2267]\", \"lift_point\": \"[0.3829, 0.2267]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3829, 0.2267]\", \"lift_point\": \"[0.3829, 0.2267]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2267]\", \"lift_point\": \"[0.3909, 0.2267]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3829, 0.2267]\", \"lift_point\": \"[0.3829, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2267]\", \"lift_point\": \"[0.3909, 0.2267]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3829, 0.2267]\", \"lift_point\": \"[0.3829, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2267]\", \"lift_point\": \"[0.3909, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6894, 0.8]\", \"lift_point\": \"[0.6894, 0.8]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3829, 0.2267]\", \"lift_point\": \"[0.3829, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3909, 0.2267]\", \"lift_point\": \"[0.3909, 0.2267]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6894, 0.8]\", \"lift_point\": \"[0.6894, 0.8]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57165_16.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57172_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.7279]\", \"lift_point\": \"[0.7714, 0.7279]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57172_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.7279]\", \"lift_point\": \"[0.7714, 0.7279]\", \"typed_text\": \"\" Goal: What is the capital of India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1152, 0.577]\", \"lift_point\": \"[0.1152, 0.577]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57172_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.7279]\", \"lift_point\": \"[0.7714, 0.7279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1152, 0.577]\", \"lift_point\": \"[0.1152, 0.577]\", \"typed_text\": \"\" Goal: What is the capital of India?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.8942]\", \"lift_point\": \"[0.1166, 0.8942]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57172_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.7279]\", \"lift_point\": \"[0.7714, 0.7279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1152, 0.577]\", \"lift_point\": \"[0.1152, 0.577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.8942]\", \"lift_point\": \"[0.1166, 0.8942]\", \"typed_text\": \"\" Goal: What is the capital of India?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of India\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57172_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.7279]\", \"lift_point\": \"[0.7714, 0.7279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1152, 0.577]\", \"lift_point\": \"[0.1152, 0.577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.8942]\", \"lift_point\": \"[0.1166, 0.8942]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of India\" Goal: What is the capital of India?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2398, 0.6228]\", \"lift_point\": \"[0.2398, 0.6228]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57172_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7714, 0.7279]\", \"lift_point\": \"[0.7714, 0.7279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1152, 0.577]\", \"lift_point\": \"[0.1152, 0.577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1166, 0.8942]\", \"lift_point\": \"[0.1166, 0.8942]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of India\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2398, 0.6228]\", \"lift_point\": \"[0.2398, 0.6228]\", \"typed_text\": \"\" Goal: What is the capital of India?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57172_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5301, 0.908]\", \"lift_point\": \"[0.5301, 0.908]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4533]\", \"lift_point\": \"[0.8903, 0.4533]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8393, 0.4724]\", \"lift_point\": \"[0.8393, 0.4724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6892, 0.7736]\", \"lift_point\": \"[0.6892, 0.7736]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6892, 0.7736]\", \"lift_point\": \"[0.6892, 0.7736]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57331_24.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.863]\", \"lift_point\": \"[0.1071, 0.863]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.863]\", \"lift_point\": \"[0.1071, 0.863]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1192, 0.0932]\", \"lift_point\": \"[0.1192, 0.0932]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.863]\", \"lift_point\": \"[0.1071, 0.863]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.8467]\", \"lift_point\": \"[0.1213, 0.8467]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0954, 0.3081]\", \"lift_point\": \"[0.0954, 0.3081]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.863]\", \"lift_point\": \"[0.1071, 0.863]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.8467]\", \"lift_point\": \"[0.1213, 0.8467]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US economy\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.863]\", \"lift_point\": \"[0.1071, 0.863]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.8467]\", \"lift_point\": \"[0.1213, 0.8467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US economy\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2597, 0.5918]\", \"lift_point\": \"[0.2597, 0.5918]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.863]\", \"lift_point\": \"[0.1071, 0.863]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.8467]\", \"lift_point\": \"[0.1213, 0.8467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US economy\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2597, 0.5918]\", \"lift_point\": \"[0.2597, 0.5918]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3296, 0.3282]\", \"lift_point\": \"[0.3296, 0.3282]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6919, 0.8615]\", \"lift_point\": \"[0.6919, 0.8615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2385, 0.567]\", \"lift_point\": \"[0.2385, 0.567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1071, 0.863]\", \"lift_point\": \"[0.1071, 0.863]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Twitter app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1213, 0.8467]\", \"lift_point\": \"[0.1213, 0.8467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news about the US economy\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2597, 0.5918]\", \"lift_point\": \"[0.2597, 0.5918]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3296, 0.3282]\", \"lift_point\": \"[0.3296, 0.3282]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57450_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6987, 0.8377]\", \"lift_point\": \"[0.6987, 0.8377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6987, 0.8377]\", \"lift_point\": \"[0.6987, 0.8377]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6987, 0.8377]\", \"lift_point\": \"[0.6987, 0.8377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6987, 0.8377]\", \"lift_point\": \"[0.6987, 0.8377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1046, 0.7265]\", \"lift_point\": \"[0.1046, 0.7265]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7137, 0.8583]\", \"lift_point\": \"[0.7137, 0.8583]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6987, 0.8377]\", \"lift_point\": \"[0.6987, 0.8377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1046, 0.7265]\", \"lift_point\": \"[0.1046, 0.7265]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Katy Perry video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2432, 0.6758]\", \"lift_point\": \"[0.2432, 0.6758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6987, 0.8377]\", \"lift_point\": \"[0.6987, 0.8377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1046, 0.7265]\", \"lift_point\": \"[0.1046, 0.7265]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Katy Perry video\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1996, 0.6082]\", \"lift_point\": \"[0.1996, 0.6082]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1245, 0.8849]\", \"lift_point\": \"[0.1245, 0.8849]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6987, 0.8377]\", \"lift_point\": \"[0.6987, 0.8377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.108, 0.1234]\", \"lift_point\": \"[0.108, 0.1234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1046, 0.7265]\", \"lift_point\": \"[0.1046, 0.7265]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Katy Perry video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1996, 0.6082]\", \"lift_point\": \"[0.1996, 0.6082]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57472_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57629_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4793, 0.6657]\", \"lift_point\": \"[0.4793, 0.6657]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57629_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4793, 0.6657]\", \"lift_point\": \"[0.4793, 0.6657]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1059, 0.7245]\", \"lift_point\": \"[0.1059, 0.7245]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57629_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4793, 0.6657]\", \"lift_point\": \"[0.4793, 0.6657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1059, 0.7245]\", \"lift_point\": \"[0.1059, 0.7245]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.9286]\", \"lift_point\": \"[0.0566, 0.9286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57629_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4793, 0.6657]\", \"lift_point\": \"[0.4793, 0.6657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1059, 0.7245]\", \"lift_point\": \"[0.1059, 0.7245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.9286]\", \"lift_point\": \"[0.0566, 0.9286]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in new jersey\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57629_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4793, 0.6657]\", \"lift_point\": \"[0.4793, 0.6657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1059, 0.7245]\", \"lift_point\": \"[0.1059, 0.7245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.9286]\", \"lift_point\": \"[0.0566, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in new jersey\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1265, 0.636]\", \"lift_point\": \"[0.1265, 0.636]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57629_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4793, 0.6657]\", \"lift_point\": \"[0.4793, 0.6657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1059, 0.7245]\", \"lift_point\": \"[0.1059, 0.7245]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0566, 0.9286]\", \"lift_point\": \"[0.0566, 0.9286]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in new jersey\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1265, 0.636]\", \"lift_point\": \"[0.1265, 0.636]\", \"typed_text\": \"\" Goal: What's a good restaurant in New Jersey?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57629_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8902, 0.4375]\", \"lift_point\": \"[0.8902, 0.4375]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57946_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8902, 0.4375]\", \"lift_point\": \"[0.8902, 0.4375]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new eyeshadow\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57946_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8902, 0.4375]\", \"lift_point\": \"[0.8902, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new eyeshadow\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.3814]\", \"lift_point\": \"[0.1197, 0.3814]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57946_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8902, 0.4375]\", \"lift_point\": \"[0.8902, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new eyeshadow\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1197, 0.3814]\", \"lift_point\": \"[0.1197, 0.3814]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57946_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.1448]\", \"lift_point\": \"[0.6476, 0.1448]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.1448]\", \"lift_point\": \"[0.6476, 0.1448]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0345, 0.5234]\", \"lift_point\": \"[0.0345, 0.5234]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.1448]\", \"lift_point\": \"[0.6476, 0.1448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0345, 0.5234]\", \"lift_point\": \"[0.0345, 0.5234]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9353]\", \"lift_point\": \"[0.0606, 0.9353]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.1448]\", \"lift_point\": \"[0.6476, 0.1448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0345, 0.5234]\", \"lift_point\": \"[0.0345, 0.5234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9353]\", \"lift_point\": \"[0.0606, 0.9353]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Seattle?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.1448]\", \"lift_point\": \"[0.6476, 0.1448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0345, 0.5234]\", \"lift_point\": \"[0.0345, 0.5234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9353]\", \"lift_point\": \"[0.0606, 0.9353]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Seattle?\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.3904]\", \"lift_point\": \"[0.1264, 0.3904]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8126, 0.5033]\", \"lift_point\": \"[0.8126, 0.5033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6476, 0.1448]\", \"lift_point\": \"[0.6476, 0.1448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0345, 0.5234]\", \"lift_point\": \"[0.0345, 0.5234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9353]\", \"lift_point\": \"[0.0606, 0.9353]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Seattle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1264, 0.3904]\", \"lift_point\": \"[0.1264, 0.3904]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Seattle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/57969_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3736, 0.1125]\", \"lift_point\": \"[0.3736, 0.1125]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3736, 0.1125]\", \"lift_point\": \"[0.3736, 0.1125]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.9447]\", \"lift_point\": \"[0.907, 0.9447]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3736, 0.1125]\", \"lift_point\": \"[0.3736, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.9447]\", \"lift_point\": \"[0.907, 0.9447]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8698, 0.5392]\", \"lift_point\": \"[0.8698, 0.5392]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3736, 0.1125]\", \"lift_point\": \"[0.3736, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.9447]\", \"lift_point\": \"[0.907, 0.9447]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8698, 0.5392]\", \"lift_point\": \"[0.8698, 0.5392]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.0588]\", \"lift_point\": \"[0.0541, 0.0588]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3736, 0.1125]\", \"lift_point\": \"[0.3736, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.9447]\", \"lift_point\": \"[0.907, 0.9447]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8698, 0.5392]\", \"lift_point\": \"[0.8698, 0.5392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.0588]\", \"lift_point\": \"[0.0541, 0.0588]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2896, 0.1464]\", \"lift_point\": \"[0.2896, 0.1464]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5611, 0.6412]\", \"lift_point\": \"[0.5611, 0.6412]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3736, 0.1125]\", \"lift_point\": \"[0.3736, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.907, 0.9447]\", \"lift_point\": \"[0.907, 0.9447]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8698, 0.5392]\", \"lift_point\": \"[0.8698, 0.5392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0541, 0.0588]\", \"lift_point\": \"[0.0541, 0.0588]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2896, 0.1464]\", \"lift_point\": \"[0.2896, 0.1464]\", \"typed_text\": \"\" Goal: Open the calendar and show me this week's events?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58006_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in London?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8884, 0.4564]\", \"lift_point\": \"[0.8884, 0.4564]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58059_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8884, 0.4564]\", \"lift_point\": \"[0.8884, 0.4564]\", \"typed_text\": \"\" Goal: What time is it in London?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in London?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58059_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8884, 0.4564]\", \"lift_point\": \"[0.8884, 0.4564]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in London?\" Goal: What time is it in London?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1262, 0.4956]\", \"lift_point\": \"[0.1262, 0.4956]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58059_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8884, 0.4564]\", \"lift_point\": \"[0.8884, 0.4564]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What time is it in London?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1262, 0.4956]\", \"lift_point\": \"[0.1262, 0.4956]\", \"typed_text\": \"\" Goal: What time is it in London?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58059_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5236, 0.8549]\", \"lift_point\": \"[0.5236, 0.8549]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5236, 0.8549]\", \"lift_point\": \"[0.5236, 0.8549]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4669]\", \"lift_point\": \"[0.0686, 0.4669]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5236, 0.8549]\", \"lift_point\": \"[0.5236, 0.8549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4669]\", \"lift_point\": \"[0.0686, 0.4669]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best burger restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5236, 0.8549]\", \"lift_point\": \"[0.5236, 0.8549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4669]\", \"lift_point\": \"[0.0686, 0.4669]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best burger restaurants\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6558]\", \"lift_point\": \"[0.7998, 0.6558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5236, 0.8549]\", \"lift_point\": \"[0.5236, 0.8549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4669]\", \"lift_point\": \"[0.0686, 0.4669]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best burger restaurants\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0685, 0.4157]\", \"lift_point\": \"[0.0685, 0.4157]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5236, 0.8549]\", \"lift_point\": \"[0.5236, 0.8549]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.4669]\", \"lift_point\": \"[0.0686, 0.4669]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"best burger restaurants\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58310_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7978, 0.662]\", \"lift_point\": \"[0.7978, 0.662]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7978, 0.662]\", \"lift_point\": \"[0.7978, 0.662]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.723, 0.453]\", \"lift_point\": \"[0.723, 0.453]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7978, 0.662]\", \"lift_point\": \"[0.7978, 0.662]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.723, 0.453]\", \"lift_point\": \"[0.723, 0.453]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6733]\", \"lift_point\": \"[0.7817, 0.6733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7978, 0.662]\", \"lift_point\": \"[0.7978, 0.662]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.723, 0.453]\", \"lift_point\": \"[0.723, 0.453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5654, 0.8708]\", \"lift_point\": \"[0.5654, 0.8708]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0743, 0.4867]\", \"lift_point\": \"[0.0743, 0.4867]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7978, 0.662]\", \"lift_point\": \"[0.7978, 0.662]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.723, 0.453]\", \"lift_point\": \"[0.723, 0.453]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5654, 0.8708]\", \"lift_point\": \"[0.5654, 0.8708]\", \"typed_text\": \"\" Goal: Search for a new mascara on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58378_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58508_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.6831]\", \"lift_point\": \"[0.7946, 0.6831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58508_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.6831]\", \"lift_point\": \"[0.7946, 0.6831]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.071, 0.5263]\", \"lift_point\": \"[0.071, 0.5263]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58508_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.6831]\", \"lift_point\": \"[0.7946, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.071, 0.5263]\", \"lift_point\": \"[0.071, 0.5263]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Taiwan\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58508_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.6831]\", \"lift_point\": \"[0.7946, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.071, 0.5263]\", \"lift_point\": \"[0.071, 0.5263]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Taiwan\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58508_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.6831]\", \"lift_point\": \"[0.7946, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.071, 0.5263]\", \"lift_point\": \"[0.071, 0.5263]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Taiwan\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2755, 0.2146]\", \"lift_point\": \"[0.2755, 0.2146]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58508_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.6831]\", \"lift_point\": \"[0.7946, 0.6831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.071, 0.5263]\", \"lift_point\": \"[0.071, 0.5263]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Taiwan\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2755, 0.2146]\", \"lift_point\": \"[0.2755, 0.2146]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58508_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3974, 0.6744]\", \"lift_point\": \"[0.3974, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.101, 0.1315]\", \"lift_point\": \"[0.101, 0.1315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5328]\", \"lift_point\": \"[0.8899, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5264, 0.7463]\", \"lift_point\": \"[0.5264, 0.7463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4094, 0.4969]\", \"lift_point\": \"[0.4094, 0.4969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3131, 0.7597]\", \"lift_point\": \"[0.3131, 0.7597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5751, 0.7483]\", \"lift_point\": \"[0.5751, 0.7483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6312, 0.5665]\", \"lift_point\": \"[0.6312, 0.5665]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4304, 0.6323]\", \"lift_point\": \"[0.4304, 0.6323]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4683, 0.7203]\", \"lift_point\": \"[0.4683, 0.7203]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4545, 0.2854]\", \"lift_point\": \"[0.4545, 0.2854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5322, 0.7355]\", \"lift_point\": \"[0.5322, 0.7355]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4545, 0.2854]\", \"lift_point\": \"[0.4545, 0.2854]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4112, 0.3784]\", \"lift_point\": \"[0.4112, 0.3784]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.7326]\", \"lift_point\": \"[0.5737, 0.7326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4545, 0.2854]\", \"lift_point\": \"[0.4545, 0.2854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4112, 0.3784]\", \"lift_point\": \"[0.4112, 0.3784]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4019, 0.5089]\", \"lift_point\": \"[0.4019, 0.5089]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6219, 0.6098]\", \"lift_point\": \"[0.6219, 0.6098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4545, 0.2854]\", \"lift_point\": \"[0.4545, 0.2854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4112, 0.3784]\", \"lift_point\": \"[0.4112, 0.3784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4019, 0.5089]\", \"lift_point\": \"[0.4019, 0.5089]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.721, 0.762]\", \"lift_point\": \"[0.721, 0.762]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6599, 0.5324]\", \"lift_point\": \"[0.6599, 0.5324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4545, 0.2854]\", \"lift_point\": \"[0.4545, 0.2854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4112, 0.3784]\", \"lift_point\": \"[0.4112, 0.3784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4019, 0.5089]\", \"lift_point\": \"[0.4019, 0.5089]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.721, 0.762]\", \"lift_point\": \"[0.721, 0.762]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8425, 0.8956]\", \"lift_point\": \"[0.8425, 0.8956]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6382, 0.3902]\", \"lift_point\": \"[0.6382, 0.3902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4545, 0.2854]\", \"lift_point\": \"[0.4545, 0.2854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4112, 0.3784]\", \"lift_point\": \"[0.4112, 0.3784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4019, 0.5089]\", \"lift_point\": \"[0.4019, 0.5089]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.721, 0.762]\", \"lift_point\": \"[0.721, 0.762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8425, 0.8956]\", \"lift_point\": \"[0.8425, 0.8956]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7761, 0.7667]\", \"lift_point\": \"[0.7761, 0.7667]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5897, 0.2898]\", \"lift_point\": \"[0.5897, 0.2898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.524, 0.262]\", \"lift_point\": \"[0.524, 0.262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4545, 0.2854]\", \"lift_point\": \"[0.4545, 0.2854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4112, 0.3784]\", \"lift_point\": \"[0.4112, 0.3784]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4019, 0.5089]\", \"lift_point\": \"[0.4019, 0.5089]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.721, 0.762]\", \"lift_point\": \"[0.721, 0.762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8425, 0.8956]\", \"lift_point\": \"[0.8425, 0.8956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7761, 0.7667]\", \"lift_point\": \"[0.7761, 0.7667]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58547_26.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6919]\", \"lift_point\": \"[0.788, 0.6919]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58624_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6919]\", \"lift_point\": \"[0.788, 0.6919]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.3622]\", \"lift_point\": \"[0.0592, 0.3622]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58624_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6919]\", \"lift_point\": \"[0.788, 0.6919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.3622]\", \"lift_point\": \"[0.0592, 0.3622]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in rio de janeiro\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58624_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6919]\", \"lift_point\": \"[0.788, 0.6919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.3622]\", \"lift_point\": \"[0.0592, 0.3622]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in rio de janeiro\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58624_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6919]\", \"lift_point\": \"[0.788, 0.6919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.3622]\", \"lift_point\": \"[0.0592, 0.3622]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in rio de janeiro\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58624_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.788, 0.6919]\", \"lift_point\": \"[0.788, 0.6919]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.3622]\", \"lift_point\": \"[0.0592, 0.3622]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in rio de janeiro\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Rio de Janeiro?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58624_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3739, 0.4837]\", \"lift_point\": \"[0.3739, 0.4837]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3739, 0.4837]\", \"lift_point\": \"[0.3739, 0.4837]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2041, 0.7799]\", \"lift_point\": \"[0.2041, 0.7799]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3739, 0.4837]\", \"lift_point\": \"[0.3739, 0.4837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2041, 0.7799]\", \"lift_point\": \"[0.2041, 0.7799]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.605]\", \"lift_point\": \"[0.0673, 0.605]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3739, 0.4837]\", \"lift_point\": \"[0.3739, 0.4837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2041, 0.7799]\", \"lift_point\": \"[0.2041, 0.7799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.605]\", \"lift_point\": \"[0.0673, 0.605]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.9107]\", \"lift_point\": \"[0.0605, 0.9107]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3739, 0.4837]\", \"lift_point\": \"[0.3739, 0.4837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2041, 0.7799]\", \"lift_point\": \"[0.2041, 0.7799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.605]\", \"lift_point\": \"[0.0673, 0.605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.9107]\", \"lift_point\": \"[0.0605, 0.9107]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rateagainst the Swiss Franc?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3739, 0.4837]\", \"lift_point\": \"[0.3739, 0.4837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2041, 0.7799]\", \"lift_point\": \"[0.2041, 0.7799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.605]\", \"lift_point\": \"[0.0673, 0.605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.9107]\", \"lift_point\": \"[0.0605, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rateagainst the Swiss Franc?\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.3486]\", \"lift_point\": \"[0.1419, 0.3486]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3739, 0.4837]\", \"lift_point\": \"[0.3739, 0.4837]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2041, 0.7799]\", \"lift_point\": \"[0.2041, 0.7799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.605]\", \"lift_point\": \"[0.0673, 0.605]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.9107]\", \"lift_point\": \"[0.0605, 0.9107]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the US dollar exchange rateagainst the Swiss Franc?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.3486]\", \"lift_point\": \"[0.1419, 0.3486]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58690_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58693_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.773, 0.1221]\", \"lift_point\": \"[0.773, 0.1221]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58693_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.773, 0.1221]\", \"lift_point\": \"[0.773, 0.1221]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58693_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3759, 0.4832]\", \"lift_point\": \"[0.3759, 0.4832]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3759, 0.4832]\", \"lift_point\": \"[0.3759, 0.4832]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.3441]\", \"lift_point\": \"[0.0605, 0.3441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6956, 0.6917]\", \"lift_point\": \"[0.6956, 0.6917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3759, 0.4832]\", \"lift_point\": \"[0.3759, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.3441]\", \"lift_point\": \"[0.0605, 0.3441]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0741, 0.9326]\", \"lift_point\": \"[0.0741, 0.9326]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3759, 0.4832]\", \"lift_point\": \"[0.3759, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.3441]\", \"lift_point\": \"[0.0605, 0.3441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0741, 0.9326]\", \"lift_point\": \"[0.0741, 0.9326]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Houston?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2882, 0.6028]\", \"lift_point\": \"[0.2882, 0.6028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3759, 0.4832]\", \"lift_point\": \"[0.3759, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.3441]\", \"lift_point\": \"[0.0605, 0.3441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0741, 0.9326]\", \"lift_point\": \"[0.0741, 0.9326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Houston?\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1376, 0.4405]\", \"lift_point\": \"[0.1376, 0.4405]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.9018]\", \"lift_point\": \"[0.559, 0.9018]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3759, 0.4832]\", \"lift_point\": \"[0.3759, 0.4832]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.3441]\", \"lift_point\": \"[0.0605, 0.3441]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0741, 0.9326]\", \"lift_point\": \"[0.0741, 0.9326]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Houston?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1376, 0.4405]\", \"lift_point\": \"[0.1376, 0.4405]\", \"typed_text\": \"\" Goal: What's a good restaurant in Houston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58731_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\" Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3777, 0.5038]\", \"lift_point\": \"[0.3777, 0.5038]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3777, 0.5038]\", \"lift_point\": \"[0.3777, 0.5038]\", \"typed_text\": \"\" Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.3137]\", \"lift_point\": \"[0.073, 0.3137]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3777, 0.5038]\", \"lift_point\": \"[0.3777, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.3137]\", \"lift_point\": \"[0.073, 0.3137]\", \"typed_text\": \"\" Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9117]\", \"lift_point\": \"[0.0639, 0.9117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3777, 0.5038]\", \"lift_point\": \"[0.3777, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.3137]\", \"lift_point\": \"[0.073, 0.3137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9117]\", \"lift_point\": \"[0.0639, 0.9117]\", \"typed_text\": \"\" Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Dennys?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3777, 0.5038]\", \"lift_point\": \"[0.3777, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.3137]\", \"lift_point\": \"[0.073, 0.3137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9117]\", \"lift_point\": \"[0.0639, 0.9117]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Dennys?\" Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1505, 0.3754]\", \"lift_point\": \"[0.1505, 0.3754]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5675, 0.6284]\", \"lift_point\": \"[0.5675, 0.6284]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3777, 0.5038]\", \"lift_point\": \"[0.3777, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.3137]\", \"lift_point\": \"[0.073, 0.3137]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9117]\", \"lift_point\": \"[0.0639, 0.9117]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Dennys?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1505, 0.3754]\", \"lift_point\": \"[0.1505, 0.3754]\", \"typed_text\": \"\" Goal: What's on the menu at Denny's?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58739_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1431, 0.6986]\", \"lift_point\": \"[0.1431, 0.6986]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7833, 0.6886]\", \"lift_point\": \"[0.7833, 0.6886]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1431, 0.6986]\", \"lift_point\": \"[0.1431, 0.6986]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0785, 0.4736]\", \"lift_point\": \"[0.0785, 0.4736]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6322, 0.8427]\", \"lift_point\": \"[0.6322, 0.8427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1431, 0.6986]\", \"lift_point\": \"[0.1431, 0.6986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0785, 0.4736]\", \"lift_point\": \"[0.0785, 0.4736]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.9253]\", \"lift_point\": \"[0.067, 0.9253]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7845, 0.6913]\", \"lift_point\": \"[0.7845, 0.6913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1431, 0.6986]\", \"lift_point\": \"[0.1431, 0.6986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0785, 0.4736]\", \"lift_point\": \"[0.0785, 0.4736]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.9253]\", \"lift_point\": \"[0.067, 0.9253]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1431, 0.6986]\", \"lift_point\": \"[0.1431, 0.6986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0785, 0.4736]\", \"lift_point\": \"[0.0785, 0.4736]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.9253]\", \"lift_point\": \"[0.067, 0.9253]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1417, 0.3989]\", \"lift_point\": \"[0.1417, 0.3989]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.3091]\", \"lift_point\": \"[0.0654, 0.3091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0682, 0.9154]\", \"lift_point\": \"[0.0682, 0.9154]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1431, 0.6986]\", \"lift_point\": \"[0.1431, 0.6986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0785, 0.4736]\", \"lift_point\": \"[0.0785, 0.4736]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.067, 0.9253]\", \"lift_point\": \"[0.067, 0.9253]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Zurich\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1417, 0.3989]\", \"lift_point\": \"[0.1417, 0.3989]\", \"typed_text\": \"\" Goal: Search for hotels in Zurich>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58802_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the calculator>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58839_1.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3775, 0.51]\", \"lift_point\": \"[0.3775, 0.51]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3775, 0.51]\", \"lift_point\": \"[0.3775, 0.51]\", \"typed_text\": \"\" Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5193]\", \"lift_point\": \"[0.3556, 0.5193]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3775, 0.51]\", \"lift_point\": \"[0.3775, 0.51]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5193]\", \"lift_point\": \"[0.3556, 0.5193]\", \"typed_text\": \"\" Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0733, 0.3744]\", \"lift_point\": \"[0.0733, 0.3744]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3775, 0.51]\", \"lift_point\": \"[0.3775, 0.51]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5193]\", \"lift_point\": \"[0.3556, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0733, 0.3744]\", \"lift_point\": \"[0.0733, 0.3744]\", \"typed_text\": \"\" Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0747, 0.9215]\", \"lift_point\": \"[0.0747, 0.9215]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3775, 0.51]\", \"lift_point\": \"[0.3775, 0.51]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5193]\", \"lift_point\": \"[0.3556, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0733, 0.3744]\", \"lift_point\": \"[0.0733, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0747, 0.9215]\", \"lift_point\": \"[0.0747, 0.9215]\", \"typed_text\": \"\" Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at In-N-Out?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3775, 0.51]\", \"lift_point\": \"[0.3775, 0.51]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5193]\", \"lift_point\": \"[0.3556, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0733, 0.3744]\", \"lift_point\": \"[0.0733, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0747, 0.9215]\", \"lift_point\": \"[0.0747, 0.9215]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at In-N-Out?\" Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.5146]\", \"lift_point\": \"[0.1491, 0.5146]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3775, 0.51]\", \"lift_point\": \"[0.3775, 0.51]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3556, 0.5193]\", \"lift_point\": \"[0.3556, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0733, 0.3744]\", \"lift_point\": \"[0.0733, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0747, 0.9215]\", \"lift_point\": \"[0.0747, 0.9215]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at In-N-Out?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.5146]\", \"lift_point\": \"[0.1491, 0.5146]\", \"typed_text\": \"\" Goal: What's on the menu at In-N-Out?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58847_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3378]\", \"lift_point\": \"[0.062, 0.3378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3378]\", \"lift_point\": \"[0.062, 0.3378]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.9254]\", \"lift_point\": \"[0.0757, 0.9254]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3378]\", \"lift_point\": \"[0.062, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.9254]\", \"lift_point\": \"[0.0757, 0.9254]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3378]\", \"lift_point\": \"[0.062, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.9254]\", \"lift_point\": \"[0.0757, 0.9254]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1533, 0.502]\", \"lift_point\": \"[0.1533, 0.502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3378]\", \"lift_point\": \"[0.062, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.9254]\", \"lift_point\": \"[0.0757, 0.9254]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1533, 0.502]\", \"lift_point\": \"[0.1533, 0.502]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.5009]\", \"lift_point\": \"[0.1423, 0.5009]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3652, 0.5103]\", \"lift_point\": \"[0.3652, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.3378]\", \"lift_point\": \"[0.062, 0.3378]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0757, 0.9254]\", \"lift_point\": \"[0.0757, 0.9254]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good Korean restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1533, 0.502]\", \"lift_point\": \"[0.1533, 0.502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.5009]\", \"lift_point\": \"[0.1423, 0.5009]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/58871_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.858, 0.6033]\", \"lift_point\": \"[0.858, 0.6033]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0441, 0.3452]\", \"lift_point\": \"[0.0441, 0.3452]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Hong Kong?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.3367]\", \"lift_point\": \"[0.1159, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9574, 0.4087]\", \"lift_point\": \"[0.9574, 0.4087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.48, 0.4451]\", \"lift_point\": \"[0.48, 0.4451]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2723, 0.6483]\", \"lift_point\": \"[0.2723, 0.6483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4632, 0.4748]\", \"lift_point\": \"[0.4632, 0.4748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9477, 0.388]\", \"lift_point\": \"[0.9477, 0.388]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2628, 0.6309]\", \"lift_point\": \"[0.2628, 0.6309]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2139, 0.6602]\", \"lift_point\": \"[0.2139, 0.6602]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_54.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_55.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_56.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_57.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_58.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_59.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_60.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_61.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_62.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_63.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_64.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_65.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_66.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_67.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_68.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_69.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_70.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_71.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_72.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2883, 0.3108]\", \"lift_point\": \"[0.2883, 0.3108]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_73.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_74.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_75.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_76.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_77.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_78.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_79.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4509, 0.4779]\", \"lift_point\": \"[0.4509, 0.4779]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_80.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4509, 0.4779]\", \"lift_point\": \"[0.4509, 0.4779]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_81.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4509, 0.4779]\", \"lift_point\": \"[0.4509, 0.4779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_82.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4509, 0.4779]\", \"lift_point\": \"[0.4509, 0.4779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6051, 0.4422]\", \"lift_point\": \"[0.6051, 0.4422]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_83.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4509, 0.4779]\", \"lift_point\": \"[0.4509, 0.4779]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6051, 0.4422]\", \"lift_point\": \"[0.6051, 0.4422]\", \"typed_text\": \"\" Goal: What's the weather like in Hong Kong?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59353_84.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4109]\", \"lift_point\": \"[0.0525, 0.4109]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of France?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6834, 0.1475]\", \"lift_point\": \"[0.6834, 0.1475]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6729, 0.321]\", \"lift_point\": \"[0.6729, 0.321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6606, 0.6509]\", \"lift_point\": \"[0.6606, 0.6509]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1015, 0.551]\", \"lift_point\": \"[0.1015, 0.551]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9445, 0.8241]\", \"lift_point\": \"[0.9445, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2204, 0.6182]\", \"lift_point\": \"[0.2204, 0.6182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2324, 0.6434]\", \"lift_point\": \"[0.2324, 0.6434]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2324, 0.6434]\", \"lift_point\": \"[0.2324, 0.6434]\", \"typed_text\": \"\" Goal: What is the capital of France?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59443_52.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4985, 0.6804]\", \"lift_point\": \"[0.4985, 0.6804]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4985, 0.6804]\", \"lift_point\": \"[0.4985, 0.6804]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.7161]\", \"lift_point\": \"[0.0577, 0.7161]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4985, 0.6804]\", \"lift_point\": \"[0.4985, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.7161]\", \"lift_point\": \"[0.0577, 0.7161]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.8898]\", \"lift_point\": \"[0.0712, 0.8898]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4985, 0.6804]\", \"lift_point\": \"[0.4985, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.7161]\", \"lift_point\": \"[0.0577, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.8898]\", \"lift_point\": \"[0.0712, 0.8898]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.4163]\", \"lift_point\": \"[0.0692, 0.4163]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4985, 0.6804]\", \"lift_point\": \"[0.4985, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.7161]\", \"lift_point\": \"[0.0577, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.8898]\", \"lift_point\": \"[0.0712, 0.8898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.4163]\", \"lift_point\": \"[0.0692, 0.4163]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4985, 0.6804]\", \"lift_point\": \"[0.4985, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.7161]\", \"lift_point\": \"[0.0577, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.8898]\", \"lift_point\": \"[0.0712, 0.8898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.4163]\", \"lift_point\": \"[0.0692, 0.4163]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes? \" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2151, 0.4389]\", \"lift_point\": \"[0.2151, 0.4389]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4985, 0.6804]\", \"lift_point\": \"[0.4985, 0.6804]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0577, 0.7161]\", \"lift_point\": \"[0.0577, 0.7161]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.8898]\", \"lift_point\": \"[0.0712, 0.8898]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.4163]\", \"lift_point\": \"[0.0692, 0.4163]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the price of a 12 ladder at Lowes? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2151, 0.4389]\", \"lift_point\": \"[0.2151, 0.4389]\", \"typed_text\": \"\" Goal: What is the price of a 12' ladder at Lowes?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59657_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.9156]\", \"lift_point\": \"[0.0689, 0.9156]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.9156]\", \"lift_point\": \"[0.0689, 0.9156]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.3175]\", \"lift_point\": \"[0.0483, 0.3175]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.9156]\", \"lift_point\": \"[0.0689, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.3175]\", \"lift_point\": \"[0.0483, 0.3175]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Philippines? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.9156]\", \"lift_point\": \"[0.0689, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.3175]\", \"lift_point\": \"[0.0483, 0.3175]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Philippines? \" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4067]\", \"lift_point\": \"[0.1341, 0.4067]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.9156]\", \"lift_point\": \"[0.0689, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.3175]\", \"lift_point\": \"[0.0483, 0.3175]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Philippines? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4067]\", \"lift_point\": \"[0.1341, 0.4067]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5572, 0.8201]\", \"lift_point\": \"[0.5572, 0.8201]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.9156]\", \"lift_point\": \"[0.0689, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.3175]\", \"lift_point\": \"[0.0483, 0.3175]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Philippines? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4067]\", \"lift_point\": \"[0.1341, 0.4067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5572, 0.8201]\", \"lift_point\": \"[0.5572, 0.8201]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1623, 0.3842]\", \"lift_point\": \"[0.1623, 0.3842]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4953, 0.7194]\", \"lift_point\": \"[0.4953, 0.7194]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.7481]\", \"lift_point\": \"[0.0552, 0.7481]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.9156]\", \"lift_point\": \"[0.0689, 0.9156]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0483, 0.3175]\", \"lift_point\": \"[0.0483, 0.3175]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Philippines? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1341, 0.4067]\", \"lift_point\": \"[0.1341, 0.4067]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5572, 0.8201]\", \"lift_point\": \"[0.5572, 0.8201]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1623, 0.3842]\", \"lift_point\": \"[0.1623, 0.3842]\", \"typed_text\": \"\" Goal: What's the news in the Philippines?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/59748_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6252, 0.3348]\", \"lift_point\": \"[0.6252, 0.3348]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60169_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6252, 0.3348]\", \"lift_point\": \"[0.6252, 0.3348]\", \"typed_text\": \"\" Goal: What's on the menu at Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6386, 0.2705]\", \"lift_point\": \"[0.6386, 0.2705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60169_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6252, 0.3348]\", \"lift_point\": \"[0.6252, 0.3348]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6386, 0.2705]\", \"lift_point\": \"[0.6386, 0.2705]\", \"typed_text\": \"\" Goal: What's on the menu at Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6386, 0.2705]\", \"lift_point\": \"[0.6386, 0.2705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60169_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6252, 0.3348]\", \"lift_point\": \"[0.6252, 0.3348]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6386, 0.2705]\", \"lift_point\": \"[0.6386, 0.2705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6386, 0.2705]\", \"lift_point\": \"[0.6386, 0.2705]\", \"typed_text\": \"\" Goal: What's on the menu at Burger King?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3922, 0.5348]\", \"lift_point\": \"[0.3922, 0.5348]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60169_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6252, 0.3348]\", \"lift_point\": \"[0.6252, 0.3348]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6386, 0.2705]\", \"lift_point\": \"[0.6386, 0.2705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6386, 0.2705]\", \"lift_point\": \"[0.6386, 0.2705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3922, 0.5348]\", \"lift_point\": \"[0.3922, 0.5348]\", \"typed_text\": \"\" Goal: What's on the menu at Burger King?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60169_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2223, 0.7321]\", \"lift_point\": \"[0.2223, 0.7321]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2223, 0.7321]\", \"lift_point\": \"[0.2223, 0.7321]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9074]\", \"lift_point\": \"[0.0708, 0.9074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2223, 0.7321]\", \"lift_point\": \"[0.2223, 0.7321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9074]\", \"lift_point\": \"[0.0708, 0.9074]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.074, 0.3732]\", \"lift_point\": \"[0.074, 0.3732]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2223, 0.7321]\", \"lift_point\": \"[0.2223, 0.7321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9074]\", \"lift_point\": \"[0.0708, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.074, 0.3732]\", \"lift_point\": \"[0.074, 0.3732]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2223, 0.7321]\", \"lift_point\": \"[0.2223, 0.7321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9074]\", \"lift_point\": \"[0.0708, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.074, 0.3732]\", \"lift_point\": \"[0.074, 0.3732]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city \" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.3762]\", \"lift_point\": \"[0.1612, 0.3762]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2223, 0.7321]\", \"lift_point\": \"[0.2223, 0.7321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9074]\", \"lift_point\": \"[0.0708, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.074, 0.3732]\", \"lift_point\": \"[0.074, 0.3732]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.3762]\", \"lift_point\": \"[0.1612, 0.3762]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3079, 0.428]\", \"lift_point\": \"[0.3079, 0.428]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6161, 0.3105]\", \"lift_point\": \"[0.6161, 0.3105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2223, 0.7321]\", \"lift_point\": \"[0.2223, 0.7321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.9074]\", \"lift_point\": \"[0.0708, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.074, 0.3732]\", \"lift_point\": \"[0.074, 0.3732]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1612, 0.3762]\", \"lift_point\": \"[0.1612, 0.3762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3079, 0.428]\", \"lift_point\": \"[0.3079, 0.428]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60314_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.3315]\", \"lift_point\": \"[0.1364, 0.3315]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.3315]\", \"lift_point\": \"[0.1364, 0.3315]\", \"typed_text\": \"\" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9041]\", \"lift_point\": \"[0.0598, 0.9041]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.3315]\", \"lift_point\": \"[0.1364, 0.3315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9041]\", \"lift_point\": \"[0.0598, 0.9041]\", \"typed_text\": \"\" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.4485]\", \"lift_point\": \"[0.0588, 0.4485]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.3315]\", \"lift_point\": \"[0.1364, 0.3315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9041]\", \"lift_point\": \"[0.0598, 0.9041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.4485]\", \"lift_point\": \"[0.0588, 0.4485]\", \"typed_text\": \"\" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US stock market? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.3315]\", \"lift_point\": \"[0.1364, 0.3315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9041]\", \"lift_point\": \"[0.0598, 0.9041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.4485]\", \"lift_point\": \"[0.0588, 0.4485]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US stock market? \" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1404, 0.3738]\", \"lift_point\": \"[0.1404, 0.3738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.3315]\", \"lift_point\": \"[0.1364, 0.3315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9041]\", \"lift_point\": \"[0.0598, 0.9041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.4485]\", \"lift_point\": \"[0.0588, 0.4485]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US stock market? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1404, 0.3738]\", \"lift_point\": \"[0.1404, 0.3738]\", \"typed_text\": \"\" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5156, 0.3897]\", \"lift_point\": \"[0.5156, 0.3897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.633, 0.2733]\", \"lift_point\": \"[0.633, 0.2733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1364, 0.3315]\", \"lift_point\": \"[0.1364, 0.3315]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.9041]\", \"lift_point\": \"[0.0598, 0.9041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0588, 0.4485]\", \"lift_point\": \"[0.0588, 0.4485]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US stock market? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1404, 0.3738]\", \"lift_point\": \"[0.1404, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5156, 0.3897]\", \"lift_point\": \"[0.5156, 0.3897]\", \"typed_text\": \"\" Goal: What's the news about the US stock market?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60410_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the gallery>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60414_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60414_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5969, 0.4606]\", \"lift_point\": \"[0.5969, 0.4606]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60414_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5969, 0.4606]\", \"lift_point\": \"[0.5969, 0.4606]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60414_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8077, 0.6506]\", \"lift_point\": \"[0.8077, 0.6506]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2986, 0.4424]\", \"lift_point\": \"[0.2986, 0.4424]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2954, 0.5354]\", \"lift_point\": \"[0.2954, 0.5354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest walmart\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.128, 0.3125]\", \"lift_point\": \"[0.128, 0.3125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3463, 0.2764]\", \"lift_point\": \"[0.3463, 0.2764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3463, 0.2764]\", \"lift_point\": \"[0.3463, 0.2764]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3463, 0.2764]\", \"lift_point\": \"[0.3463, 0.2764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3463, 0.2764]\", \"lift_point\": \"[0.3463, 0.2764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3463, 0.2764]\", \"lift_point\": \"[0.3463, 0.2764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3088, 0.6094]\", \"lift_point\": \"[0.3088, 0.6094]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.389, 0.3822]\", \"lift_point\": \"[0.389, 0.3822]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3463, 0.2764]\", \"lift_point\": \"[0.3463, 0.2764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3088, 0.6094]\", \"lift_point\": \"[0.3088, 0.6094]\", \"typed_text\": \"\" Goal: How do I get to the nearest Walmart?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/60909_20.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.9153]\", \"lift_point\": \"[0.0623, 0.9153]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.9153]\", \"lift_point\": \"[0.0623, 0.9153]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.9153]\", \"lift_point\": \"[0.0623, 0.9153]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8015, 0.6568]\", \"lift_point\": \"[0.8015, 0.6568]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.9153]\", \"lift_point\": \"[0.0623, 0.9153]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8015, 0.6568]\", \"lift_point\": \"[0.8015, 0.6568]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.9153]\", \"lift_point\": \"[0.0623, 0.9153]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8015, 0.6568]\", \"lift_point\": \"[0.8015, 0.6568]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.7097]\", \"lift_point\": \"[0.7895, 0.7097]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.9153]\", \"lift_point\": \"[0.0623, 0.9153]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8015, 0.6568]\", \"lift_point\": \"[0.8015, 0.6568]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1904, 0.5908]\", \"lift_point\": \"[0.1904, 0.5908]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4926]\", \"lift_point\": \"[0.0598, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.9153]\", \"lift_point\": \"[0.0623, 0.9153]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8015, 0.6568]\", \"lift_point\": \"[0.8015, 0.6568]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1904, 0.5908]\", \"lift_point\": \"[0.1904, 0.5908]\", \"typed_text\": \"\" Goal: Open a new Chrome window>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61181_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9182]\", \"lift_point\": \"[0.0639, 0.9182]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9182]\", \"lift_point\": \"[0.0639, 0.9182]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from san diego to seatle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9182]\", \"lift_point\": \"[0.0639, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from san diego to seatle\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1482, 0.6046]\", \"lift_point\": \"[0.1482, 0.6046]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9182]\", \"lift_point\": \"[0.0639, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from san diego to seatle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1482, 0.6046]\", \"lift_point\": \"[0.1482, 0.6046]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9182]\", \"lift_point\": \"[0.0639, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from san diego to seatle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1482, 0.6046]\", \"lift_point\": \"[0.1482, 0.6046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9182]\", \"lift_point\": \"[0.0639, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from san diego to seatle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1482, 0.6046]\", \"lift_point\": \"[0.1482, 0.6046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.446]\", \"lift_point\": \"[0.8495, 0.446]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7817, 0.6517]\", \"lift_point\": \"[0.7817, 0.6517]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.3969]\", \"lift_point\": \"[0.0675, 0.3969]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0639, 0.9182]\", \"lift_point\": \"[0.0639, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from san diego to seatle\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1482, 0.6046]\", \"lift_point\": \"[0.1482, 0.6046]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8495, 0.446]\", \"lift_point\": \"[0.8495, 0.446]\", \"typed_text\": \"\" Goal: Search for flights from San Diego to Seattle>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61263_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6448]\", \"lift_point\": \"[0.7931, 0.6448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7925, 0.6805]\", \"lift_point\": \"[0.7925, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0674, 0.8177]\", \"lift_point\": \"[0.0674, 0.8177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1732, 0.9098]\", \"lift_point\": \"[0.1732, 0.9098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.892]\", \"lift_point\": \"[0.173, 0.892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1765, 0.9076]\", \"lift_point\": \"[0.1765, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [TYPE,PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1752, 0.9076]\", \"lift_point\": \"[0.1752, 0.9076]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1508, 0.9043]\", \"lift_point\": \"[0.1508, 0.9043]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1766, 0.8965]\", \"lift_point\": \"[0.1766, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1792, 0.8965]\", \"lift_point\": \"[0.1792, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.9208]\", \"lift_point\": \"[0.1278, 0.9208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0601, 0.9113]\", \"lift_point\": \"[0.0601, 0.9113]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.6001]\", \"lift_point\": \"[0.331, 0.6001]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3087, 0.6682]\", \"lift_point\": \"[0.3087, 0.6682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"latest news in space exploration\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3964, 0.5443]\", \"lift_point\": \"[0.3964, 0.5443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2154, 0.5227]\", \"lift_point\": \"[0.2154, 0.5227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3964, 0.5443]\", \"lift_point\": \"[0.3964, 0.5443]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3964, 0.5443]\", \"lift_point\": \"[0.3964, 0.5443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4624]\", \"lift_point\": \"[0.3323, 0.4624]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6425, 0.3476]\", \"lift_point\": \"[0.6425, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3964, 0.5443]\", \"lift_point\": \"[0.3964, 0.5443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3323, 0.4624]\", \"lift_point\": \"[0.3323, 0.4624]\", \"typed_text\": \"\" Goal: What's the latest news in space exploration?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61293_30.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8006, 0.6824]\", \"lift_point\": \"[0.8006, 0.6824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0726, 0.4245]\", \"lift_point\": \"[0.0726, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the new in bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.1962]\", \"lift_point\": \"[0.559, 0.1962]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1326, 0.4855]\", \"lift_point\": \"[0.1326, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.1962]\", \"lift_point\": \"[0.559, 0.1962]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.857, 0.9552]\", \"lift_point\": \"[0.857, 0.9552]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6276, 0.6021]\", \"lift_point\": \"[0.6276, 0.6021]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.1962]\", \"lift_point\": \"[0.559, 0.1962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.857, 0.9552]\", \"lift_point\": \"[0.857, 0.9552]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8529, 0.9696]\", \"lift_point\": \"[0.8529, 0.9696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.1962]\", \"lift_point\": \"[0.559, 0.1962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.857, 0.9552]\", \"lift_point\": \"[0.857, 0.9552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8542, 0.972]\", \"lift_point\": \"[0.8542, 0.972]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.1962]\", \"lift_point\": \"[0.559, 0.1962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.857, 0.9552]\", \"lift_point\": \"[0.857, 0.9552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.1962]\", \"lift_point\": \"[0.559, 0.1962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.857, 0.9552]\", \"lift_point\": \"[0.857, 0.9552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4715, 0.9252]\", \"lift_point\": \"[0.4715, 0.9252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.559, 0.1962]\", \"lift_point\": \"[0.559, 0.1962]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.857, 0.9552]\", \"lift_point\": \"[0.857, 0.9552]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4715, 0.9252]\", \"lift_point\": \"[0.4715, 0.9252]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61351_18.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.9072]\", \"lift_point\": \"[0.0564, 0.9072]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.9072]\", \"lift_point\": \"[0.0564, 0.9072]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Mexico\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.9072]\", \"lift_point\": \"[0.0564, 0.9072]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Mexico\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.2502]\", \"lift_point\": \"[0.1329, 0.2502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.9072]\", \"lift_point\": \"[0.0564, 0.9072]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Mexico\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.2502]\", \"lift_point\": \"[0.1329, 0.2502]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5498, 0.2842]\", \"lift_point\": \"[0.5498, 0.2842]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.9072]\", \"lift_point\": \"[0.0564, 0.9072]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Mexico\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.2502]\", \"lift_point\": \"[0.1329, 0.2502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5498, 0.2842]\", \"lift_point\": \"[0.5498, 0.2842]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7309, 0.5486]\", \"lift_point\": \"[0.7309, 0.5486]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.9072]\", \"lift_point\": \"[0.0564, 0.9072]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Mexico\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.2502]\", \"lift_point\": \"[0.1329, 0.2502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5498, 0.2842]\", \"lift_point\": \"[0.5498, 0.2842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7309, 0.5486]\", \"lift_point\": \"[0.7309, 0.5486]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7324, 0.8548]\", \"lift_point\": \"[0.7324, 0.8548]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4808, 0.7102]\", \"lift_point\": \"[0.4808, 0.7102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.3981]\", \"lift_point\": \"[0.1319, 0.3981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0564, 0.9072]\", \"lift_point\": \"[0.0564, 0.9072]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of Mexico\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1329, 0.2502]\", \"lift_point\": \"[0.1329, 0.2502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5498, 0.2842]\", \"lift_point\": \"[0.5498, 0.2842]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7309, 0.5486]\", \"lift_point\": \"[0.7309, 0.5486]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7324, 0.8548]\", \"lift_point\": \"[0.7324, 0.8548]\", \"typed_text\": \"\" Goal: Google the capital of Mexico>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61549_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4927, 0.6819]\", \"lift_point\": \"[0.4927, 0.6819]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4927, 0.6819]\", \"lift_point\": \"[0.4927, 0.6819]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.5854]\", \"lift_point\": \"[0.12, 0.5854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4927, 0.6819]\", \"lift_point\": \"[0.4927, 0.6819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.5854]\", \"lift_point\": \"[0.12, 0.5854]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0631, 0.9282]\", \"lift_point\": \"[0.0631, 0.9282]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4927, 0.6819]\", \"lift_point\": \"[0.4927, 0.6819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.5854]\", \"lift_point\": \"[0.12, 0.5854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0631, 0.9282]\", \"lift_point\": \"[0.0631, 0.9282]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4927, 0.6819]\", \"lift_point\": \"[0.4927, 0.6819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.5854]\", \"lift_point\": \"[0.12, 0.5854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0631, 0.9282]\", \"lift_point\": \"[0.0631, 0.9282]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.3715]\", \"lift_point\": \"[0.1211, 0.3715]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4927, 0.6819]\", \"lift_point\": \"[0.4927, 0.6819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.5854]\", \"lift_point\": \"[0.12, 0.5854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0631, 0.9282]\", \"lift_point\": \"[0.0631, 0.9282]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.3715]\", \"lift_point\": \"[0.1211, 0.3715]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2598, 0.2621]\", \"lift_point\": \"[0.2598, 0.2621]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4927, 0.6819]\", \"lift_point\": \"[0.4927, 0.6819]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.5854]\", \"lift_point\": \"[0.12, 0.5854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0631, 0.9282]\", \"lift_point\": \"[0.0631, 0.9282]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.3715]\", \"lift_point\": \"[0.1211, 0.3715]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2598, 0.2621]\", \"lift_point\": \"[0.2598, 0.2621]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61609_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4801, 0.6961]\", \"lift_point\": \"[0.4801, 0.6961]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4801, 0.6961]\", \"lift_point\": \"[0.4801, 0.6961]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2509, 0.5219]\", \"lift_point\": \"[0.2509, 0.5219]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4801, 0.6961]\", \"lift_point\": \"[0.4801, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2509, 0.5219]\", \"lift_point\": \"[0.2509, 0.5219]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.9237]\", \"lift_point\": \"[0.0593, 0.9237]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4801, 0.6961]\", \"lift_point\": \"[0.4801, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2509, 0.5219]\", \"lift_point\": \"[0.2509, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.9237]\", \"lift_point\": \"[0.0593, 0.9237]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4801, 0.6961]\", \"lift_point\": \"[0.4801, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2509, 0.5219]\", \"lift_point\": \"[0.2509, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.9237]\", \"lift_point\": \"[0.0593, 0.9237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco?\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.3941]\", \"lift_point\": \"[0.1294, 0.3941]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4801, 0.6961]\", \"lift_point\": \"[0.4801, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2509, 0.5219]\", \"lift_point\": \"[0.2509, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.9237]\", \"lift_point\": \"[0.0593, 0.9237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.3941]\", \"lift_point\": \"[0.1294, 0.3941]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2854, 0.4765]\", \"lift_point\": \"[0.2854, 0.4765]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4801, 0.6961]\", \"lift_point\": \"[0.4801, 0.6961]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2509, 0.5219]\", \"lift_point\": \"[0.2509, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.9237]\", \"lift_point\": \"[0.0593, 0.9237]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.3941]\", \"lift_point\": \"[0.1294, 0.3941]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2854, 0.4765]\", \"lift_point\": \"[0.2854, 0.4765]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61631_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.6774]\", \"lift_point\": \"[0.4399, 0.6774]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.6774]\", \"lift_point\": \"[0.4399, 0.6774]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2615, 0.5066]\", \"lift_point\": \"[0.2615, 0.5066]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.6774]\", \"lift_point\": \"[0.4399, 0.6774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2615, 0.5066]\", \"lift_point\": \"[0.2615, 0.5066]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.9251]\", \"lift_point\": \"[0.0582, 0.9251]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.6774]\", \"lift_point\": \"[0.4399, 0.6774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2615, 0.5066]\", \"lift_point\": \"[0.2615, 0.5066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.9251]\", \"lift_point\": \"[0.0582, 0.9251]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.6774]\", \"lift_point\": \"[0.4399, 0.6774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2615, 0.5066]\", \"lift_point\": \"[0.2615, 0.5066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.9251]\", \"lift_point\": \"[0.0582, 0.9251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3828]\", \"lift_point\": \"[0.1279, 0.3828]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.6774]\", \"lift_point\": \"[0.4399, 0.6774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2615, 0.5066]\", \"lift_point\": \"[0.2615, 0.5066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.9251]\", \"lift_point\": \"[0.0582, 0.9251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3828]\", \"lift_point\": \"[0.1279, 0.3828]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2896, 0.4649]\", \"lift_point\": \"[0.2896, 0.4649]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4399, 0.6774]\", \"lift_point\": \"[0.4399, 0.6774]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2615, 0.5066]\", \"lift_point\": \"[0.2615, 0.5066]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.9251]\", \"lift_point\": \"[0.0582, 0.9251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.3828]\", \"lift_point\": \"[0.1279, 0.3828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2896, 0.4649]\", \"lift_point\": \"[0.2896, 0.4649]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61675_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8979, 0.4573]\", \"lift_point\": \"[0.8979, 0.4573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61809_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8979, 0.4573]\", \"lift_point\": \"[0.8979, 0.4573]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61809_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8979, 0.4573]\", \"lift_point\": \"[0.8979, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0916, 0.6115]\", \"lift_point\": \"[0.0916, 0.6115]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61809_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8979, 0.4573]\", \"lift_point\": \"[0.8979, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the new iPad cost?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0916, 0.6115]\", \"lift_point\": \"[0.0916, 0.6115]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61809_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9121, 0.5982]\", \"lift_point\": \"[0.9121, 0.5982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61924_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9121, 0.5982]\", \"lift_point\": \"[0.9121, 0.5982]\", \"typed_text\": \"\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bangladesh?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61924_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9121, 0.5982]\", \"lift_point\": \"[0.9121, 0.5982]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bangladesh?\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.349]\", \"lift_point\": \"[0.116, 0.349]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61924_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9121, 0.5982]\", \"lift_point\": \"[0.9121, 0.5982]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bangladesh?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.116, 0.349]\", \"lift_point\": \"[0.116, 0.349]\", \"typed_text\": \"\" Goal: What's the news in Bangladesh?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61924_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.3636]\", \"lift_point\": \"[0.9053, 0.3636]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61964_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.3636]\", \"lift_point\": \"[0.9053, 0.3636]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the menu at IHOP\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61964_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.3636]\", \"lift_point\": \"[0.9053, 0.3636]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the menu at IHOP\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1138, 0.3252]\", \"lift_point\": \"[0.1138, 0.3252]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61964_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9053, 0.3636]\", \"lift_point\": \"[0.9053, 0.3636]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the menu at IHOP\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1138, 0.3252]\", \"lift_point\": \"[0.1138, 0.3252]\", \"typed_text\": \"\" Goal: What's on the menu at IHOP?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61964_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9177, 0.4904]\", \"lift_point\": \"[0.9177, 0.4904]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61974_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9177, 0.4904]\", \"lift_point\": \"[0.9177, 0.4904]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61974_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9177, 0.4904]\", \"lift_point\": \"[0.9177, 0.4904]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store?\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5647]\", \"lift_point\": \"[0.118, 0.5647]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61974_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9177, 0.4904]\", \"lift_point\": \"[0.9177, 0.4904]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest T-Mobile Store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5647]\", \"lift_point\": \"[0.118, 0.5647]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/61974_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8528]\", \"lift_point\": \"[0.0519, 0.8528]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8528]\", \"lift_point\": \"[0.0519, 0.8528]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.4626]\", \"lift_point\": \"[0.0568, 0.4626]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8384, 0.573]\", \"lift_point\": \"[0.8384, 0.573]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8528]\", \"lift_point\": \"[0.0519, 0.8528]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.4626]\", \"lift_point\": \"[0.0568, 0.4626]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.0685]\", \"lift_point\": \"[0.0521, 0.0685]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0603, 0.0385]\", \"lift_point\": \"[0.0603, 0.0385]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0556]\", \"lift_point\": \"[0.0531, 0.0556]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0519, 0.8528]\", \"lift_point\": \"[0.0519, 0.8528]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0568, 0.4626]\", \"lift_point\": \"[0.0568, 0.4626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0521, 0.0685]\", \"lift_point\": \"[0.0521, 0.0685]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62045_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How far is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.918, 0.5556]\", \"lift_point\": \"[0.918, 0.5556]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62063_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.918, 0.5556]\", \"lift_point\": \"[0.918, 0.5556]\", \"typed_text\": \"\" Goal: How far is the moon?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How far is the moon\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62063_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.918, 0.5556]\", \"lift_point\": \"[0.918, 0.5556]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How far is the moon\" Goal: How far is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.5415]\", \"lift_point\": \"[0.1176, 0.5415]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62063_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.918, 0.5556]\", \"lift_point\": \"[0.918, 0.5556]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How far is the moon\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1176, 0.5415]\", \"lift_point\": \"[0.1176, 0.5415]\", \"typed_text\": \"\" Goal: How far is the moon?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62063_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62079_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.6196]\", \"lift_point\": \"[0.8934, 0.6196]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62079_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.6196]\", \"lift_point\": \"[0.8934, 0.6196]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the president of the United States?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62079_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.6196]\", \"lift_point\": \"[0.8934, 0.6196]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the president of the United States?\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1177, 0.4356]\", \"lift_point\": \"[0.1177, 0.4356]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62079_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.6196]\", \"lift_point\": \"[0.8934, 0.6196]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the president of the United States?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1177, 0.4356]\", \"lift_point\": \"[0.1177, 0.4356]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62079_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62183_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62183_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62183_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" president of the United States\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62183_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" president of the United States\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62183_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8013, 0.1648]\", \"lift_point\": \"[0.8013, 0.1648]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" president of the United States\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62183_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new blush>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62224_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.1718]\", \"lift_point\": \"[0.8868, 0.1718]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62224_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.1718]\", \"lift_point\": \"[0.8868, 0.1718]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.2336]\", \"lift_point\": \"[0.8847, 0.2336]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62224_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.1718]\", \"lift_point\": \"[0.8868, 0.1718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.2336]\", \"lift_point\": \"[0.8847, 0.2336]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new blush\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62224_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.1718]\", \"lift_point\": \"[0.8868, 0.1718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.2336]\", \"lift_point\": \"[0.8847, 0.2336]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new blush\" Goal: Search for a new blush>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62224_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.1718]\", \"lift_point\": \"[0.8868, 0.1718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.2336]\", \"lift_point\": \"[0.8847, 0.2336]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new blush\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.4572]\", \"lift_point\": \"[0.3263, 0.4572]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62224_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.1718]\", \"lift_point\": \"[0.8868, 0.1718]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.2336]\", \"lift_point\": \"[0.8847, 0.2336]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new blush\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3263, 0.4572]\", \"lift_point\": \"[0.3263, 0.4572]\", \"typed_text\": \"\" Goal: Search for a new blush>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62224_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62378_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5686]\", \"lift_point\": \"[0.8899, 0.5686]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62378_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5686]\", \"lift_point\": \"[0.8899, 0.5686]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the president of the United States\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62378_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5686]\", \"lift_point\": \"[0.8899, 0.5686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the president of the United States\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62378_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8899, 0.5686]\", \"lift_point\": \"[0.8899, 0.5686]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Who is the president of the United States\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Who is the president of the United States?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62378_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5124, 0.5403]\", \"lift_point\": \"[0.5124, 0.5403]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5124, 0.5403]\", \"lift_point\": \"[0.5124, 0.5403]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8828, 0.6863]\", \"lift_point\": \"[0.8828, 0.6863]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.5339]\", \"lift_point\": \"[0.8925, 0.5339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new eyeshadow\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5124, 0.5403]\", \"lift_point\": \"[0.5124, 0.5403]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8828, 0.6863]\", \"lift_point\": \"[0.8828, 0.6863]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62436_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62459_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.878, 0.4599]\", \"lift_point\": \"[0.878, 0.4599]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62459_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.878, 0.4599]\", \"lift_point\": \"[0.878, 0.4599]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Sprint Store\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62459_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.878, 0.4599]\", \"lift_point\": \"[0.878, 0.4599]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Sprint Store\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62459_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.878, 0.4599]\", \"lift_point\": \"[0.878, 0.4599]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Sprint Store\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Sprint Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62459_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.3061]\", \"lift_point\": \"[0.895, 0.3061]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.3061]\", \"lift_point\": \"[0.895, 0.3061]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.3061]\", \"lift_point\": \"[0.895, 0.3061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.3061]\", \"lift_point\": \"[0.895, 0.3061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3382, 0.4826]\", \"lift_point\": \"[0.3382, 0.4826]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.3061]\", \"lift_point\": \"[0.895, 0.3061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3382, 0.4826]\", \"lift_point\": \"[0.3382, 0.4826]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.7443]\", \"lift_point\": \"[0.8047, 0.7443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.3061]\", \"lift_point\": \"[0.895, 0.3061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3382, 0.4826]\", \"lift_point\": \"[0.3382, 0.4826]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.7443]\", \"lift_point\": \"[0.8047, 0.7443]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.9417]\", \"lift_point\": \"[0.6461, 0.9417]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.3061]\", \"lift_point\": \"[0.895, 0.3061]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Orlando\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3382, 0.4826]\", \"lift_point\": \"[0.3382, 0.4826]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8047, 0.7443]\", \"lift_point\": \"[0.8047, 0.7443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6461, 0.9417]\", \"lift_point\": \"[0.6461, 0.9417]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62491_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6491, 0.4777]\", \"lift_point\": \"[0.6491, 0.4777]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8832, 0.4339]\", \"lift_point\": \"[0.8832, 0.4339]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news about the US economy\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6491, 0.4777]\", \"lift_point\": \"[0.6491, 0.4777]\", \"typed_text\": \"\" Goal: What's the news about the US economy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62521_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7179, 0.4779]\", \"lift_point\": \"[0.7179, 0.4779]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3866]\", \"lift_point\": \"[0.8814, 0.3866]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Tokyo\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7179, 0.4779]\", \"lift_point\": \"[0.7179, 0.4779]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Tokyo>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62557_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.5997]\", \"lift_point\": \"[0.8847, 0.5997]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62563_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.5997]\", \"lift_point\": \"[0.8847, 0.5997]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62563_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.5997]\", \"lift_point\": \"[0.8847, 0.5997]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Philadelphia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62563_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.5997]\", \"lift_point\": \"[0.8847, 0.5997]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Philadelphia\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62563_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.5997]\", \"lift_point\": \"[0.8847, 0.5997]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Philadelphia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.353, 0.463]\", \"lift_point\": \"[0.353, 0.463]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62563_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8847, 0.5997]\", \"lift_point\": \"[0.8847, 0.5997]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Philadelphia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.353, 0.463]\", \"lift_point\": \"[0.353, 0.463]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Philadelphia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62563_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4375]\", \"lift_point\": \"[0.8975, 0.4375]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62576_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4375]\", \"lift_point\": \"[0.8975, 0.4375]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62576_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4375]\", \"lift_point\": \"[0.8975, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1697, 0.5376]\", \"lift_point\": \"[0.1697, 0.5376]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62576_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4375]\", \"lift_point\": \"[0.8975, 0.4375]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1697, 0.5376]\", \"lift_point\": \"[0.1697, 0.5376]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62576_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new highlighter>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62605_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new highlighter>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9068, 0.4789]\", \"lift_point\": \"[0.9068, 0.4789]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62605_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9068, 0.4789]\", \"lift_point\": \"[0.9068, 0.4789]\", \"typed_text\": \"\" Goal: Search for a new highlighter>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new highlighter\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62605_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9068, 0.4789]\", \"lift_point\": \"[0.9068, 0.4789]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new highlighter\" Goal: Search for a new highlighter>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1187, 0.3716]\", \"lift_point\": \"[0.1187, 0.3716]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62605_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9068, 0.4789]\", \"lift_point\": \"[0.9068, 0.4789]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new highlighter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1187, 0.3716]\", \"lift_point\": \"[0.1187, 0.3716]\", \"typed_text\": \"\" Goal: Search for a new highlighter>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62605_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How old is the earth?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62628_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How old is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.5157]\", \"lift_point\": \"[0.8922, 0.5157]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62628_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.5157]\", \"lift_point\": \"[0.8922, 0.5157]\", \"typed_text\": \"\" Goal: How old is the earth?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How old is the earth?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62628_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.5157]\", \"lift_point\": \"[0.8922, 0.5157]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How old is the earth?\" Goal: How old is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1773, 0.4353]\", \"lift_point\": \"[0.1773, 0.4353]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62628_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8922, 0.5157]\", \"lift_point\": \"[0.8922, 0.5157]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How old is the earth?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1773, 0.4353]\", \"lift_point\": \"[0.1773, 0.4353]\", \"typed_text\": \"\" Goal: How old is the earth?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62628_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8978, 0.4062]\", \"lift_point\": \"[0.8978, 0.4062]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62654_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8978, 0.4062]\", \"lift_point\": \"[0.8978, 0.4062]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Switzerland?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62654_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8978, 0.4062]\", \"lift_point\": \"[0.8978, 0.4062]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Switzerland?\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.4396]\", \"lift_point\": \"[0.1189, 0.4396]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62654_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8978, 0.4062]\", \"lift_point\": \"[0.8978, 0.4062]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Switzerland?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1189, 0.4396]\", \"lift_point\": \"[0.1189, 0.4396]\", \"typed_text\": \"\" Goal: What is the capital of Switzerland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62654_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6534, 0.5577]\", \"lift_point\": \"[0.6534, 0.5577]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6534, 0.5577]\", \"lift_point\": \"[0.6534, 0.5577]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6534, 0.5577]\", \"lift_point\": \"[0.6534, 0.5577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6534, 0.5577]\", \"lift_point\": \"[0.6534, 0.5577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new eyeliner\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8404, 0.5427]\", \"lift_point\": \"[0.8404, 0.5427]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6534, 0.5577]\", \"lift_point\": \"[0.6534, 0.5577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new eyeliner\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.2958]\", \"lift_point\": \"[0.1211, 0.2958]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6534, 0.5577]\", \"lift_point\": \"[0.6534, 0.5577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new eyeliner\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.2958]\", \"lift_point\": \"[0.1211, 0.2958]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.2992]\", \"lift_point\": \"[0.1642, 0.2992]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.485, 0.32]\", \"lift_point\": \"[0.485, 0.32]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.555, 0.3808]\", \"lift_point\": \"[0.555, 0.3808]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6534, 0.5577]\", \"lift_point\": \"[0.6534, 0.5577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.4337]\", \"lift_point\": \"[0.8915, 0.4337]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new eyeliner\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.2958]\", \"lift_point\": \"[0.1211, 0.2958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.2992]\", \"lift_point\": \"[0.1642, 0.2992]\", \"typed_text\": \"\" Goal: Search for a new eyeliner>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62700_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated burger restaurants on Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62973_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.8735]\", \"lift_point\": \"[0.7763, 0.8735]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62973_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.8735]\", \"lift_point\": \"[0.7763, 0.8735]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2932]\", \"lift_point\": \"[0.059, 0.2932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62973_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.8735]\", \"lift_point\": \"[0.7763, 0.8735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2932]\", \"lift_point\": \"[0.059, 0.2932]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62973_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.8735]\", \"lift_point\": \"[0.7763, 0.8735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2932]\", \"lift_point\": \"[0.059, 0.2932]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants\" Goal: Search for top rated burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1459, 0.5262]\", \"lift_point\": \"[0.1459, 0.5262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62973_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7763, 0.8735]\", \"lift_point\": \"[0.7763, 0.8735]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2932]\", \"lift_point\": \"[0.059, 0.2932]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1459, 0.5262]\", \"lift_point\": \"[0.1459, 0.5262]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/62973_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.2611]\", \"lift_point\": \"[0.133, 0.2611]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.2611]\", \"lift_point\": \"[0.133, 0.2611]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.151, 0.8536]\", \"lift_point\": \"[0.151, 0.8536]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.2611]\", \"lift_point\": \"[0.133, 0.2611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.151, 0.8536]\", \"lift_point\": \"[0.151, 0.8536]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.84]\", \"lift_point\": \"[0.1519, 0.84]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.2611]\", \"lift_point\": \"[0.133, 0.2611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.151, 0.8536]\", \"lift_point\": \"[0.151, 0.8536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.84]\", \"lift_point\": \"[0.1519, 0.84]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6924, 0.4806]\", \"lift_point\": \"[0.6924, 0.4806]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.8754]\", \"lift_point\": \"[0.7864, 0.8754]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.2611]\", \"lift_point\": \"[0.133, 0.2611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.151, 0.8536]\", \"lift_point\": \"[0.151, 0.8536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.84]\", \"lift_point\": \"[0.1519, 0.84]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6924, 0.4806]\", \"lift_point\": \"[0.6924, 0.4806]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.8912]\", \"lift_point\": \"[0.0721, 0.8912]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.7847]\", \"lift_point\": \"[0.0615, 0.7847]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.2611]\", \"lift_point\": \"[0.133, 0.2611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.151, 0.8536]\", \"lift_point\": \"[0.151, 0.8536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.84]\", \"lift_point\": \"[0.1519, 0.84]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6924, 0.4806]\", \"lift_point\": \"[0.6924, 0.4806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.8912]\", \"lift_point\": \"[0.0721, 0.8912]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4201, 0.261]\", \"lift_point\": \"[0.4201, 0.261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"google play books app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.133, 0.2611]\", \"lift_point\": \"[0.133, 0.2611]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.151, 0.8536]\", \"lift_point\": \"[0.151, 0.8536]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.84]\", \"lift_point\": \"[0.1519, 0.84]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6924, 0.4806]\", \"lift_point\": \"[0.6924, 0.4806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.8912]\", \"lift_point\": \"[0.0721, 0.8912]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4201, 0.261]\", \"lift_point\": \"[0.4201, 0.261]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Books app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63015_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6893]\", \"lift_point\": \"[0.7768, 0.6893]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6893]\", \"lift_point\": \"[0.7768, 0.6893]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.3666]\", \"lift_point\": \"[0.046, 0.3666]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6893]\", \"lift_point\": \"[0.7768, 0.6893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.3666]\", \"lift_point\": \"[0.046, 0.3666]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in suriname?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6893]\", \"lift_point\": \"[0.7768, 0.6893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.3666]\", \"lift_point\": \"[0.046, 0.3666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in suriname?\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.2373]\", \"lift_point\": \"[0.1234, 0.2373]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6893]\", \"lift_point\": \"[0.7768, 0.6893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.3666]\", \"lift_point\": \"[0.046, 0.3666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in suriname?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.2373]\", \"lift_point\": \"[0.1234, 0.2373]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3658, 0.1658]\", \"lift_point\": \"[0.3658, 0.1658]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6893]\", \"lift_point\": \"[0.7768, 0.6893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.3666]\", \"lift_point\": \"[0.046, 0.3666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in suriname?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.2373]\", \"lift_point\": \"[0.1234, 0.2373]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3658, 0.1658]\", \"lift_point\": \"[0.3658, 0.1658]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8344, 0.2127]\", \"lift_point\": \"[0.8344, 0.2127]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7768, 0.6893]\", \"lift_point\": \"[0.7768, 0.6893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.046, 0.3666]\", \"lift_point\": \"[0.046, 0.3666]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in suriname?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.2373]\", \"lift_point\": \"[0.1234, 0.2373]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3658, 0.1658]\", \"lift_point\": \"[0.3658, 0.1658]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8344, 0.2127]\", \"lift_point\": \"[0.8344, 0.2127]\", \"typed_text\": \"\" Goal: What's the news in Suriname?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63023_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How far is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6906]\", \"lift_point\": \"[0.7789, 0.6906]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63036_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6906]\", \"lift_point\": \"[0.7789, 0.6906]\", \"typed_text\": \"\" Goal: How far is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.2967]\", \"lift_point\": \"[0.0479, 0.2967]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63036_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6906]\", \"lift_point\": \"[0.7789, 0.6906]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.2967]\", \"lift_point\": \"[0.0479, 0.2967]\", \"typed_text\": \"\" Goal: How far is the moon?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How far is the moon?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63036_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6906]\", \"lift_point\": \"[0.7789, 0.6906]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.2967]\", \"lift_point\": \"[0.0479, 0.2967]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How far is the moon?\" Goal: How far is the moon?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.481]\", \"lift_point\": \"[0.124, 0.481]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63036_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6906]\", \"lift_point\": \"[0.7789, 0.6906]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.2967]\", \"lift_point\": \"[0.0479, 0.2967]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How far is the moon?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.481]\", \"lift_point\": \"[0.124, 0.481]\", \"typed_text\": \"\" Goal: How far is the moon?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63036_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6805]\", \"lift_point\": \"[0.7825, 0.6805]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63127_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6805]\", \"lift_point\": \"[0.7825, 0.6805]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4198]\", \"lift_point\": \"[0.0464, 0.4198]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63127_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6805]\", \"lift_point\": \"[0.7825, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4198]\", \"lift_point\": \"[0.0464, 0.4198]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for my computer\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63127_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6805]\", \"lift_point\": \"[0.7825, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4198]\", \"lift_point\": \"[0.0464, 0.4198]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for my computer\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1226, 0.3679]\", \"lift_point\": \"[0.1226, 0.3679]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63127_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6805]\", \"lift_point\": \"[0.7825, 0.6805]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.4198]\", \"lift_point\": \"[0.0464, 0.4198]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for my computer\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1226, 0.3679]\", \"lift_point\": \"[0.1226, 0.3679]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my computer>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63127_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6899]\", \"lift_point\": \"[0.785, 0.6899]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63156_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6899]\", \"lift_point\": \"[0.785, 0.6899]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.4535]\", \"lift_point\": \"[0.0448, 0.4535]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63156_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6899]\", \"lift_point\": \"[0.785, 0.6899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.4535]\", \"lift_point\": \"[0.0448, 0.4535]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63156_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6899]\", \"lift_point\": \"[0.785, 0.6899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.4535]\", \"lift_point\": \"[0.0448, 0.4535]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.3068]\", \"lift_point\": \"[0.1278, 0.3068]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63156_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6899]\", \"lift_point\": \"[0.785, 0.6899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.4535]\", \"lift_point\": \"[0.0448, 0.4535]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.3068]\", \"lift_point\": \"[0.1278, 0.3068]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.3758]\", \"lift_point\": \"[0.3782, 0.3758]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63156_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6899]\", \"lift_point\": \"[0.785, 0.6899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.4535]\", \"lift_point\": \"[0.0448, 0.4535]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.3068]\", \"lift_point\": \"[0.1278, 0.3068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.3758]\", \"lift_point\": \"[0.3782, 0.3758]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3963, 0.9217]\", \"lift_point\": \"[0.3963, 0.9217]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63156_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6899]\", \"lift_point\": \"[0.785, 0.6899]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.4535]\", \"lift_point\": \"[0.0448, 0.4535]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Philadelphia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.3068]\", \"lift_point\": \"[0.1278, 0.3068]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3782, 0.3758]\", \"lift_point\": \"[0.3782, 0.3758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3963, 0.9217]\", \"lift_point\": \"[0.3963, 0.9217]\", \"typed_text\": \"\" Goal: Search for hotels in Philadelphia>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63156_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Checkthe settings for the Amazon Prime Music app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63224_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Checkthe settings for the Amazon Prime Music app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7688, 0.1281]\", \"lift_point\": \"[0.7688, 0.1281]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63224_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7688, 0.1281]\", \"lift_point\": \"[0.7688, 0.1281]\", \"typed_text\": \"\" Goal: Checkthe settings for the Amazon Prime Music app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7191, 0.5092]\", \"lift_point\": \"[0.7191, 0.5092]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63224_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7688, 0.1281]\", \"lift_point\": \"[0.7688, 0.1281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7191, 0.5092]\", \"lift_point\": \"[0.7191, 0.5092]\", \"typed_text\": \"\" Goal: Checkthe settings for the Amazon Prime Music app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4623, 0.522]\", \"lift_point\": \"[0.4623, 0.522]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63224_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7688, 0.1281]\", \"lift_point\": \"[0.7688, 0.1281]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7191, 0.5092]\", \"lift_point\": \"[0.7191, 0.5092]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4623, 0.522]\", \"lift_point\": \"[0.4623, 0.522]\", \"typed_text\": \"\" Goal: Checkthe settings for the Amazon Prime Music app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63224_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\" Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4079]\", \"lift_point\": \"[0.0721, 0.4079]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4079]\", \"lift_point\": \"[0.0721, 0.4079]\", \"typed_text\": \"\" Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"home depot\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4079]\", \"lift_point\": \"[0.0721, 0.4079]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"home depot\" Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4079]\", \"lift_point\": \"[0.0721, 0.4079]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"home depot\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4079]\", \"lift_point\": \"[0.0721, 0.4079]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"home depot\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5978, 0.3387]\", \"lift_point\": \"[0.5978, 0.3387]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7589, 0.3169]\", \"lift_point\": \"[0.7589, 0.3169]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0721, 0.4079]\", \"lift_point\": \"[0.0721, 0.4079]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"home depot\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5978, 0.3387]\", \"lift_point\": \"[0.5978, 0.3387]\", \"typed_text\": \"\" Goal: What's the price of the 2x4x8 boards at Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63250_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like inToronto?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5971, 0.519]\", \"lift_point\": \"[0.5971, 0.519]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63465_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5971, 0.519]\", \"lift_point\": \"[0.5971, 0.519]\", \"typed_text\": \"\" Goal: What's the weather like inToronto?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.347]\", \"lift_point\": \"[0.7529, 0.347]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63465_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5971, 0.519]\", \"lift_point\": \"[0.5971, 0.519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.347]\", \"lift_point\": \"[0.7529, 0.347]\", \"typed_text\": \"\" Goal: What's the weather like inToronto?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4097]\", \"lift_point\": \"[0.3952, 0.4097]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63465_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5971, 0.519]\", \"lift_point\": \"[0.5971, 0.519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.347]\", \"lift_point\": \"[0.7529, 0.347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4097]\", \"lift_point\": \"[0.3952, 0.4097]\", \"typed_text\": \"\" Goal: What's the weather like inToronto?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in toronto\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63465_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5971, 0.519]\", \"lift_point\": \"[0.5971, 0.519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.347]\", \"lift_point\": \"[0.7529, 0.347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4097]\", \"lift_point\": \"[0.3952, 0.4097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in toronto\" Goal: What's the weather like inToronto?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63465_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5971, 0.519]\", \"lift_point\": \"[0.5971, 0.519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.347]\", \"lift_point\": \"[0.7529, 0.347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4097]\", \"lift_point\": \"[0.3952, 0.4097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in toronto\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like inToronto?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63465_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5971, 0.519]\", \"lift_point\": \"[0.5971, 0.519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.347]\", \"lift_point\": \"[0.7529, 0.347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3952, 0.4097]\", \"lift_point\": \"[0.3952, 0.4097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"weather in toronto\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like inToronto?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63465_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.2992]\", \"lift_point\": \"[0.7474, 0.2992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0719, 0.6064]\", \"lift_point\": \"[0.0719, 0.6064]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"3 bedroom apartment rent in Dallas\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6434, 0.371]\", \"lift_point\": \"[0.6434, 0.371]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.606, 0.3559]\", \"lift_point\": \"[0.606, 0.3559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3683, 0.9673]\", \"lift_point\": \"[0.3683, 0.9673]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1525, 0.0525]\", \"lift_point\": \"[0.1525, 0.0525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1544, 0.0272]\", \"lift_point\": \"[0.1544, 0.0272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5687, 0.5023]\", \"lift_point\": \"[0.5687, 0.5023]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3748, 0.9512]\", \"lift_point\": \"[0.3748, 0.9512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1544, 0.0272]\", \"lift_point\": \"[0.1544, 0.0272]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63634_27.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7482, 0.3056]\", \"lift_point\": \"[0.7482, 0.3056]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0812, 0.4505]\", \"lift_point\": \"[0.0812, 0.4505]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7474, 0.1525]\", \"lift_point\": \"[0.7474, 0.1525]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7046, 0.4999]\", \"lift_point\": \"[0.7046, 0.4999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4458, 0.4515]\", \"lift_point\": \"[0.4458, 0.4515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5768, 0.3786]\", \"lift_point\": \"[0.5768, 0.3786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5469, 0.3295]\", \"lift_point\": \"[0.5469, 0.3295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.866, 0.2635]\", \"lift_point\": \"[0.866, 0.2635]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.866, 0.2635]\", \"lift_point\": \"[0.866, 0.2635]\", \"typed_text\": \"\" Goal: Check the settings for the Google Maps app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63804_28.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my laptop>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7648, 0.308]\", \"lift_point\": \"[0.7648, 0.308]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63832_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7648, 0.308]\", \"lift_point\": \"[0.7648, 0.308]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my laptop>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0811, 0.5569]\", \"lift_point\": \"[0.0811, 0.5569]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63832_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7648, 0.308]\", \"lift_point\": \"[0.7648, 0.308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0811, 0.5569]\", \"lift_point\": \"[0.0811, 0.5569]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my laptop>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for laptop\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63832_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7648, 0.308]\", \"lift_point\": \"[0.7648, 0.308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0811, 0.5569]\", \"lift_point\": \"[0.0811, 0.5569]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for laptop\" Goal: Show me some nice wallpapers for my laptop>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63832_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7648, 0.308]\", \"lift_point\": \"[0.7648, 0.308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0811, 0.5569]\", \"lift_point\": \"[0.0811, 0.5569]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for laptop\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my laptop>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63832_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7648, 0.308]\", \"lift_point\": \"[0.7648, 0.308]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0811, 0.5569]\", \"lift_point\": \"[0.0811, 0.5569]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nice wallpapers for laptop\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my laptop>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63832_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7531, 0.3065]\", \"lift_point\": \"[0.7531, 0.3065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63856_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7531, 0.3065]\", \"lift_point\": \"[0.7531, 0.3065]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0854, 0.4441]\", \"lift_point\": \"[0.0854, 0.4441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63856_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7531, 0.3065]\", \"lift_point\": \"[0.7531, 0.3065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0854, 0.4441]\", \"lift_point\": \"[0.0854, 0.4441]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63856_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7531, 0.3065]\", \"lift_point\": \"[0.7531, 0.3065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0854, 0.4441]\", \"lift_point\": \"[0.0854, 0.4441]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63856_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7531, 0.3065]\", \"lift_point\": \"[0.7531, 0.3065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0854, 0.4441]\", \"lift_point\": \"[0.0854, 0.4441]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63856_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7531, 0.3065]\", \"lift_point\": \"[0.7531, 0.3065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0854, 0.4441]\", \"lift_point\": \"[0.0854, 0.4441]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8995, 0.5335]\", \"lift_point\": \"[0.8995, 0.5335]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63856_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7531, 0.3065]\", \"lift_point\": \"[0.7531, 0.3065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0854, 0.4441]\", \"lift_point\": \"[0.0854, 0.4441]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8995, 0.5335]\", \"lift_point\": \"[0.8995, 0.5335]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63856_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4058, 0.1114]\", \"lift_point\": \"[0.4058, 0.1114]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4058, 0.1114]\", \"lift_point\": \"[0.4058, 0.1114]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4058, 0.1114]\", \"lift_point\": \"[0.4058, 0.1114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4058, 0.1114]\", \"lift_point\": \"[0.4058, 0.1114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.842, 0.554]\", \"lift_point\": \"[0.842, 0.554]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4058, 0.1114]\", \"lift_point\": \"[0.4058, 0.1114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.842, 0.554]\", \"lift_point\": \"[0.842, 0.554]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0819, 0.8044]\", \"lift_point\": \"[0.0819, 0.8044]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4058, 0.1114]\", \"lift_point\": \"[0.4058, 0.1114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8784, 0.9095]\", \"lift_point\": \"[0.8784, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.842, 0.554]\", \"lift_point\": \"[0.842, 0.554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0819, 0.8044]\", \"lift_point\": \"[0.0819, 0.8044]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the month?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63918_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of 1000 Watt EGO power+ snow Blower\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of 1000 Watt EGO power+ snow Blower\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of 1000 Watt EGO power+ snow Blower\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of 1000 Watt EGO power+ snow Blower\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of 1000 Watt EGO power+ snow Blower\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2266, 0.5193]\", \"lift_point\": \"[0.2266, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of 1000 Watt EGO power+ snow Blower\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3363, 0.4445]\", \"lift_point\": \"[0.3363, 0.4445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2208, 0.5351]\", \"lift_point\": \"[0.2208, 0.5351]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0789, 0.5293]\", \"lift_point\": \"[0.0789, 0.5293]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of 1000 Watt EGO power+ snow Blower\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3363, 0.4445]\", \"lift_point\": \"[0.3363, 0.4445]\", \"typed_text\": \"\" Goal: What's the price of the 1000-Watt EGO Power+ Snow Blower?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/63965_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8765, 0.5046]\", \"lift_point\": \"[0.8765, 0.5046]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64141_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8765, 0.5046]\", \"lift_point\": \"[0.8765, 0.5046]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2x4x8 board cost at Lowes\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64141_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8765, 0.5046]\", \"lift_point\": \"[0.8765, 0.5046]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2x4x8 board cost at Lowes\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.5098]\", \"lift_point\": \"[0.1308, 0.5098]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64141_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8765, 0.5046]\", \"lift_point\": \"[0.8765, 0.5046]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2x4x8 board cost at Lowes\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.5098]\", \"lift_point\": \"[0.1308, 0.5098]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3837, 0.3738]\", \"lift_point\": \"[0.3837, 0.3738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64141_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8765, 0.5046]\", \"lift_point\": \"[0.8765, 0.5046]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2x4x8 board cost at Lowes\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.5098]\", \"lift_point\": \"[0.1308, 0.5098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3837, 0.3738]\", \"lift_point\": \"[0.3837, 0.3738]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64141_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8765, 0.5046]\", \"lift_point\": \"[0.8765, 0.5046]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2x4x8 board cost at Lowes\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.5098]\", \"lift_point\": \"[0.1308, 0.5098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3837, 0.3738]\", \"lift_point\": \"[0.3837, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6894, 0.462]\", \"lift_point\": \"[0.6894, 0.462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64141_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8765, 0.5046]\", \"lift_point\": \"[0.8765, 0.5046]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2x4x8 board cost at Lowes\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.5098]\", \"lift_point\": \"[0.1308, 0.5098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3837, 0.3738]\", \"lift_point\": \"[0.3837, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6894, 0.462]\", \"lift_point\": \"[0.6894, 0.462]\", \"typed_text\": \"\" Goal: How much does a 2x4x8 board cost at Lowes?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64141_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the CNN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" Goal: Install the CNN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\" Goal: Install the CNN app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.9121]\", \"lift_point\": \"[0.0565, 0.9121]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.9121]\", \"lift_point\": \"[0.0565, 0.9121]\", \"typed_text\": \"\" Goal: Install the CNN app>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"CNN app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.9121]\", \"lift_point\": \"[0.0565, 0.9121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"CNN app\" Goal: Install the CNN app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.9121]\", \"lift_point\": \"[0.0565, 0.9121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"CNN app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the CNN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.3443]\", \"lift_point\": \"[0.1365, 0.3443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.9121]\", \"lift_point\": \"[0.0565, 0.9121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"CNN app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.3443]\", \"lift_point\": \"[0.1365, 0.3443]\", \"typed_text\": \"\" Goal: Install the CNN app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5818]\", \"lift_point\": \"[0.3312, 0.5818]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.9121]\", \"lift_point\": \"[0.0565, 0.9121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"CNN app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.3443]\", \"lift_point\": \"[0.1365, 0.3443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5818]\", \"lift_point\": \"[0.3312, 0.5818]\", \"typed_text\": \"\" Goal: Install the CNN app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.365, 0.6091]\", \"lift_point\": \"[0.365, 0.6091]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7836, 0.8913]\", \"lift_point\": \"[0.7836, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.8083]\", \"lift_point\": \"[0.0641, 0.8083]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0565, 0.9121]\", \"lift_point\": \"[0.0565, 0.9121]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"CNN app\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.3443]\", \"lift_point\": \"[0.1365, 0.3443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5818]\", \"lift_point\": \"[0.3312, 0.5818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.365, 0.6091]\", \"lift_point\": \"[0.365, 0.6091]\", \"typed_text\": \"\" Goal: Install the CNN app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64212_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1722, 0.9007]\", \"lift_point\": \"[0.1722, 0.9007]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1722, 0.9007]\", \"lift_point\": \"[0.1722, 0.9007]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1703, 0.3363]\", \"lift_point\": \"[0.1703, 0.3363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1722, 0.9007]\", \"lift_point\": \"[0.1722, 0.9007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1703, 0.3363]\", \"lift_point\": \"[0.1703, 0.3363]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5092, 0.7262]\", \"lift_point\": \"[0.5092, 0.7262]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1722, 0.9007]\", \"lift_point\": \"[0.1722, 0.9007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1703, 0.3363]\", \"lift_point\": \"[0.1703, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5092, 0.7262]\", \"lift_point\": \"[0.5092, 0.7262]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.412, 0.5358]\", \"lift_point\": \"[0.412, 0.5358]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1722, 0.9007]\", \"lift_point\": \"[0.1722, 0.9007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1703, 0.3363]\", \"lift_point\": \"[0.1703, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5092, 0.7262]\", \"lift_point\": \"[0.5092, 0.7262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.412, 0.5358]\", \"lift_point\": \"[0.412, 0.5358]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3374, 0.7421]\", \"lift_point\": \"[0.3374, 0.7421]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1722, 0.9007]\", \"lift_point\": \"[0.1722, 0.9007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1703, 0.3363]\", \"lift_point\": \"[0.1703, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5092, 0.7262]\", \"lift_point\": \"[0.5092, 0.7262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.412, 0.5358]\", \"lift_point\": \"[0.412, 0.5358]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3374, 0.7421]\", \"lift_point\": \"[0.3374, 0.7421]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4095, 0.5013]\", \"lift_point\": \"[0.4095, 0.5013]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1917, 0.7347]\", \"lift_point\": \"[0.1917, 0.7347]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.077, 0.1591]\", \"lift_point\": \"[0.077, 0.1591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1722, 0.9007]\", \"lift_point\": \"[0.1722, 0.9007]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1703, 0.3363]\", \"lift_point\": \"[0.1703, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5092, 0.7262]\", \"lift_point\": \"[0.5092, 0.7262]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.412, 0.5358]\", \"lift_point\": \"[0.412, 0.5358]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3374, 0.7421]\", \"lift_point\": \"[0.3374, 0.7421]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4095, 0.5013]\", \"lift_point\": \"[0.4095, 0.5013]\", \"typed_text\": \"\" Goal: Set an alarm for 3pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64248_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64314_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3505]\", \"lift_point\": \"[0.8864, 0.3505]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64314_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3505]\", \"lift_point\": \"[0.8864, 0.3505]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64314_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3505]\", \"lift_point\": \"[0.8864, 0.3505]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64314_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3505]\", \"lift_point\": \"[0.8864, 0.3505]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1385, 0.2144]\", \"lift_point\": \"[0.1385, 0.2144]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64314_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3505]\", \"lift_point\": \"[0.8864, 0.3505]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new iPad cost\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1385, 0.2144]\", \"lift_point\": \"[0.1385, 0.2144]\", \"typed_text\": \"\" Goal: How much does the new iPad cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64314_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3613]\", \"lift_point\": \"[0.0634, 0.3613]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3613]\", \"lift_point\": \"[0.0634, 0.3613]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3613]\", \"lift_point\": \"[0.0634, 0.3613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news? \" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.4889]\", \"lift_point\": \"[0.3, 0.4889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3613]\", \"lift_point\": \"[0.0634, 0.3613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.4889]\", \"lift_point\": \"[0.3, 0.4889]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3613]\", \"lift_point\": \"[0.0634, 0.3613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.4889]\", \"lift_point\": \"[0.3, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5718, 0.3868]\", \"lift_point\": \"[0.5718, 0.3868]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5116, 0.6707]\", \"lift_point\": \"[0.5116, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3613]\", \"lift_point\": \"[0.0634, 0.3613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.4889]\", \"lift_point\": \"[0.3, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5718, 0.3868]\", \"lift_point\": \"[0.5718, 0.3868]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.845, 0.4068]\", \"lift_point\": \"[0.845, 0.4068]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.575]\", \"lift_point\": \"[0.1434, 0.575]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0735, 0.9084]\", \"lift_point\": \"[0.0735, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0634, 0.3613]\", \"lift_point\": \"[0.0634, 0.3613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3, 0.4889]\", \"lift_point\": \"[0.3, 0.4889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5718, 0.3868]\", \"lift_point\": \"[0.5718, 0.3868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.845, 0.4068]\", \"lift_point\": \"[0.845, 0.4068]\", \"typed_text\": \"\" Goal: What's the news?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64383_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new hair product>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new hair product \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new hair product \" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5502]\", \"lift_point\": \"[0.2757, 0.5502]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5316, 0.696]\", \"lift_point\": \"[0.5316, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new hair product \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5502]\", \"lift_point\": \"[0.2757, 0.5502]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3859, 0.3724]\", \"lift_point\": \"[0.3859, 0.3724]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5139, 0.6907]\", \"lift_point\": \"[0.5139, 0.6907]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new hair product \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5502]\", \"lift_point\": \"[0.2757, 0.5502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3859, 0.3724]\", \"lift_point\": \"[0.3859, 0.3724]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1398, 0.5773]\", \"lift_point\": \"[0.1398, 0.5773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new hair product \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5502]\", \"lift_point\": \"[0.2757, 0.5502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3859, 0.3724]\", \"lift_point\": \"[0.3859, 0.3724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.9012]\", \"lift_point\": \"[0.778, 0.9012]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0667, 0.8632]\", \"lift_point\": \"[0.0667, 0.8632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new hair product \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5502]\", \"lift_point\": \"[0.2757, 0.5502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3859, 0.3724]\", \"lift_point\": \"[0.3859, 0.3724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.9012]\", \"lift_point\": \"[0.778, 0.9012]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.9012]\", \"lift_point\": \"[0.7804, 0.9012]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0272, 0.5224]\", \"lift_point\": \"[0.0272, 0.5224]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0729, 0.6006]\", \"lift_point\": \"[0.0729, 0.6006]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new hair product \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2757, 0.5502]\", \"lift_point\": \"[0.2757, 0.5502]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3859, 0.3724]\", \"lift_point\": \"[0.3859, 0.3724]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.9012]\", \"lift_point\": \"[0.778, 0.9012]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.9012]\", \"lift_point\": \"[0.7804, 0.9012]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64415_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4019]\", \"lift_point\": \"[0.1414, 0.4019]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4019]\", \"lift_point\": \"[0.1414, 0.4019]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3341, 0.3463]\", \"lift_point\": \"[0.3341, 0.3463]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4019]\", \"lift_point\": \"[0.1414, 0.4019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3341, 0.3463]\", \"lift_point\": \"[0.3341, 0.3463]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5042, 0.6569]\", \"lift_point\": \"[0.5042, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4019]\", \"lift_point\": \"[0.1414, 0.4019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3341, 0.3463]\", \"lift_point\": \"[0.3341, 0.3463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.812, 0.3459]\", \"lift_point\": \"[0.812, 0.3459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1423, 0.505]\", \"lift_point\": \"[0.1423, 0.505]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4019]\", \"lift_point\": \"[0.1414, 0.4019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3341, 0.3463]\", \"lift_point\": \"[0.3341, 0.3463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.812, 0.3459]\", \"lift_point\": \"[0.812, 0.3459]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0657, 0.9022]\", \"lift_point\": \"[0.0657, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4019]\", \"lift_point\": \"[0.1414, 0.4019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3341, 0.3463]\", \"lift_point\": \"[0.3341, 0.3463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.812, 0.3459]\", \"lift_point\": \"[0.812, 0.3459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.9659]\", \"lift_point\": \"[0.779, 0.9659]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.4723]\", \"lift_point\": \"[0.0592, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1414, 0.4019]\", \"lift_point\": \"[0.1414, 0.4019]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3341, 0.3463]\", \"lift_point\": \"[0.3341, 0.3463]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.812, 0.3459]\", \"lift_point\": \"[0.812, 0.3459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.779, 0.9659]\", \"lift_point\": \"[0.779, 0.9659]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64479_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Google app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.5181]\", \"lift_point\": \"[0.0552, 0.5181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.5181]\", \"lift_point\": \"[0.0552, 0.5181]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.5181]\", \"lift_point\": \"[0.0552, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app \" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1439, 0.4632]\", \"lift_point\": \"[0.1439, 0.4632]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.5181]\", \"lift_point\": \"[0.0552, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1439, 0.4632]\", \"lift_point\": \"[0.1439, 0.4632]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2411, 0.3249]\", \"lift_point\": \"[0.2411, 0.3249]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.5181]\", \"lift_point\": \"[0.0552, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1439, 0.4632]\", \"lift_point\": \"[0.1439, 0.4632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2411, 0.3249]\", \"lift_point\": \"[0.2411, 0.3249]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2788, 0.6746]\", \"lift_point\": \"[0.2788, 0.6746]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6476]\", \"lift_point\": \"[0.6352, 0.6476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0552, 0.5181]\", \"lift_point\": \"[0.0552, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google app \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1439, 0.4632]\", \"lift_point\": \"[0.1439, 0.4632]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2411, 0.3249]\", \"lift_point\": \"[0.2411, 0.3249]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2788, 0.6746]\", \"lift_point\": \"[0.2788, 0.6746]\", \"typed_text\": \"\" Goal: Install the Google app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64593_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.9074]\", \"lift_point\": \"[0.0654, 0.9074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1211, 0.6071]\", \"lift_point\": \"[0.1211, 0.6071]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.9074]\", \"lift_point\": \"[0.0654, 0.9074]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.079, 0.4799]\", \"lift_point\": \"[0.079, 0.4799]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.9074]\", \"lift_point\": \"[0.0654, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.079, 0.4799]\", \"lift_point\": \"[0.079, 0.4799]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a giraffe \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.9074]\", \"lift_point\": \"[0.0654, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.079, 0.4799]\", \"lift_point\": \"[0.079, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a giraffe \" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.3377]\", \"lift_point\": \"[0.1352, 0.3377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4577, 0.4598]\", \"lift_point\": \"[0.4577, 0.4598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.9074]\", \"lift_point\": \"[0.0654, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.079, 0.4799]\", \"lift_point\": \"[0.079, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a giraffe \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.3377]\", \"lift_point\": \"[0.1352, 0.3377]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1439, 0.3429]\", \"lift_point\": \"[0.1439, 0.3429]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4678, 0.6666]\", \"lift_point\": \"[0.4678, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.4758]\", \"lift_point\": \"[0.1432, 0.4758]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0654, 0.9074]\", \"lift_point\": \"[0.0654, 0.9074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.079, 0.4799]\", \"lift_point\": \"[0.079, 0.4799]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a giraffe \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1352, 0.3377]\", \"lift_point\": \"[0.1352, 0.3377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1439, 0.3429]\", \"lift_point\": \"[0.1439, 0.3429]\", \"typed_text\": \"\" Goal: How big is a giraffe?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64633_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4791, 0.8649]\", \"lift_point\": \"[0.4791, 0.8649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3151, 0.5738]\", \"lift_point\": \"[0.3151, 0.5738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.9079]\", \"lift_point\": \"[0.064, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.4275]\", \"lift_point\": \"[0.0598, 0.4275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5101, 0.3663]\", \"lift_point\": \"[0.5101, 0.3663]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Portland? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5101, 0.3663]\", \"lift_point\": \"[0.5101, 0.3663]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0524]\", \"lift_point\": \"[0.0498, 0.0524]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1432, 0.3592]\", \"lift_point\": \"[0.1432, 0.3592]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5101, 0.3663]\", \"lift_point\": \"[0.5101, 0.3663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0524]\", \"lift_point\": \"[0.0498, 0.0524]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.1024]\", \"lift_point\": \"[0.0598, 0.1024]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5607, 0.7688]\", \"lift_point\": \"[0.5607, 0.7688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5101, 0.3663]\", \"lift_point\": \"[0.5101, 0.3663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0524]\", \"lift_point\": \"[0.0498, 0.0524]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.1024]\", \"lift_point\": \"[0.0598, 0.1024]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3034, 0.5706]\", \"lift_point\": \"[0.3034, 0.5706]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5101, 0.3663]\", \"lift_point\": \"[0.5101, 0.3663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0524]\", \"lift_point\": \"[0.0498, 0.0524]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.1024]\", \"lift_point\": \"[0.0598, 0.1024]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6818, 0.4711]\", \"lift_point\": \"[0.6818, 0.4711]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4668, 0.3446]\", \"lift_point\": \"[0.4668, 0.3446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.0783]\", \"lift_point\": \"[0.0618, 0.0783]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5101, 0.3663]\", \"lift_point\": \"[0.5101, 0.3663]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0524]\", \"lift_point\": \"[0.0498, 0.0524]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0598, 0.1024]\", \"lift_point\": \"[0.0598, 0.1024]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6818, 0.4711]\", \"lift_point\": \"[0.6818, 0.4711]\", \"typed_text\": \"\" Goal: What's a good restaurant in Portland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64899_19.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3217, 0.4698]\", \"lift_point\": \"[0.3217, 0.4698]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4966, 0.621]\", \"lift_point\": \"[0.4966, 0.621]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.8752]\", \"lift_point\": \"[0.7824, 0.8752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3207, 0.3363]\", \"lift_point\": \"[0.3207, 0.3363]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.6227]\", \"lift_point\": \"[0.3153, 0.6227]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.6227]\", \"lift_point\": \"[0.3153, 0.6227]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9213]\", \"lift_point\": \"[0.0724, 0.9213]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1595, 0.8726]\", \"lift_point\": \"[0.1595, 0.8726]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.6227]\", \"lift_point\": \"[0.3153, 0.6227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9213]\", \"lift_point\": \"[0.0724, 0.9213]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.4441]\", \"lift_point\": \"[0.057, 0.4441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.6227]\", \"lift_point\": \"[0.3153, 0.6227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9213]\", \"lift_point\": \"[0.0724, 0.9213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.4441]\", \"lift_point\": \"[0.057, 0.4441]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.6227]\", \"lift_point\": \"[0.3153, 0.6227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9213]\", \"lift_point\": \"[0.0724, 0.9213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.4441]\", \"lift_point\": \"[0.057, 0.4441]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1462, 0.524]\", \"lift_point\": \"[0.1462, 0.524]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7076, 0.7498]\", \"lift_point\": \"[0.7076, 0.7498]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4836, 0.8681]\", \"lift_point\": \"[0.4836, 0.8681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3153, 0.6227]\", \"lift_point\": \"[0.3153, 0.6227]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.9213]\", \"lift_point\": \"[0.0724, 0.9213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.057, 0.4441]\", \"lift_point\": \"[0.057, 0.4441]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mexico City? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1462, 0.524]\", \"lift_point\": \"[0.1462, 0.524]\", \"typed_text\": \"\" Goal: What's the weather like in Mexico City?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/64917_18.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.471, 0.8687]\", \"lift_point\": \"[0.471, 0.8687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1379, 0.4722]\", \"lift_point\": \"[0.1379, 0.4722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9085]\", \"lift_point\": \"[0.0591, 0.9085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4597]\", \"lift_point\": \"[0.0647, 0.4597]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.2228]\", \"lift_point\": \"[0.059, 0.2228]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4597]\", \"lift_point\": \"[0.0647, 0.4597]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest grocery store \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4597]\", \"lift_point\": \"[0.0647, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest grocery store \" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1676, 0.4682]\", \"lift_point\": \"[0.1676, 0.4682]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4597]\", \"lift_point\": \"[0.0647, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1676, 0.4682]\", \"lift_point\": \"[0.1676, 0.4682]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1641, 0.4303]\", \"lift_point\": \"[0.1641, 0.4303]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4597]\", \"lift_point\": \"[0.0647, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1676, 0.4682]\", \"lift_point\": \"[0.1676, 0.4682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1767, 0.7518]\", \"lift_point\": \"[0.1767, 0.7518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4597]\", \"lift_point\": \"[0.0647, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1676, 0.4682]\", \"lift_point\": \"[0.1676, 0.4682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7623, 0.601]\", \"lift_point\": \"[0.7623, 0.601]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0908, 0.5762]\", \"lift_point\": \"[0.0908, 0.5762]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0762, 0.9013]\", \"lift_point\": \"[0.0762, 0.9013]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.4597]\", \"lift_point\": \"[0.0647, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the nearest grocery store \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1676, 0.4682]\", \"lift_point\": \"[0.1676, 0.4682]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7623, 0.601]\", \"lift_point\": \"[0.7623, 0.601]\", \"typed_text\": \"\" Goal: How do I get to the nearest grocery store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65010_18.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.4817]\", \"lift_point\": \"[0.0697, 0.4817]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.4817]\", \"lift_point\": \"[0.0697, 0.4817]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.905]\", \"lift_point\": \"[0.0671, 0.905]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.4817]\", \"lift_point\": \"[0.0697, 0.4817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.905]\", \"lift_point\": \"[0.0671, 0.905]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0687, 0.3571]\", \"lift_point\": \"[0.0687, 0.3571]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.4817]\", \"lift_point\": \"[0.0697, 0.4817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.905]\", \"lift_point\": \"[0.0671, 0.905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0687, 0.3571]\", \"lift_point\": \"[0.0687, 0.3571]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.4817]\", \"lift_point\": \"[0.0697, 0.4817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.905]\", \"lift_point\": \"[0.0671, 0.905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0687, 0.3571]\", \"lift_point\": \"[0.0687, 0.3571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone \" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1593, 0.3999]\", \"lift_point\": \"[0.1593, 0.3999]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.4817]\", \"lift_point\": \"[0.0697, 0.4817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.905]\", \"lift_point\": \"[0.0671, 0.905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0687, 0.3571]\", \"lift_point\": \"[0.0687, 0.3571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1593, 0.3999]\", \"lift_point\": \"[0.1593, 0.3999]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3359, 0.4812]\", \"lift_point\": \"[0.3359, 0.4812]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4693, 0.8415]\", \"lift_point\": \"[0.4693, 0.8415]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.4817]\", \"lift_point\": \"[0.0697, 0.4817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.905]\", \"lift_point\": \"[0.0671, 0.905]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0687, 0.3571]\", \"lift_point\": \"[0.0687, 0.3571]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1593, 0.3999]\", \"lift_point\": \"[0.1593, 0.3999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3359, 0.4812]\", \"lift_point\": \"[0.3359, 0.4812]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65143_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8521]\", \"lift_point\": \"[0.4735, 0.8521]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4735, 0.8572]\", \"lift_point\": \"[0.4735, 0.8572]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4731, 0.8504]\", \"lift_point\": \"[0.4731, 0.8504]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4819, 0.9037]\", \"lift_point\": \"[0.4819, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0831, 0.5901]\", \"lift_point\": \"[0.0831, 0.5901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.9081]\", \"lift_point\": \"[0.0705, 0.9081]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0651, 0.5329]\", \"lift_point\": \"[0.0651, 0.5329]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.4719]\", \"lift_point\": \"[0.7956, 0.4719]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new lipgloss \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.4719]\", \"lift_point\": \"[0.7956, 0.4719]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1883, 0.8366]\", \"lift_point\": \"[0.1883, 0.8366]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1519, 0.4733]\", \"lift_point\": \"[0.1519, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.4719]\", \"lift_point\": \"[0.7956, 0.4719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1883, 0.8366]\", \"lift_point\": \"[0.1883, 0.8366]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7233, 0.3531]\", \"lift_point\": \"[0.7233, 0.3531]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3788, 0.3476]\", \"lift_point\": \"[0.3788, 0.3476]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.4719]\", \"lift_point\": \"[0.7956, 0.4719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1883, 0.8366]\", \"lift_point\": \"[0.1883, 0.8366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7233, 0.3531]\", \"lift_point\": \"[0.7233, 0.3531]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9086, 0.8965]\", \"lift_point\": \"[0.9086, 0.8965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.4719]\", \"lift_point\": \"[0.7956, 0.4719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1883, 0.8366]\", \"lift_point\": \"[0.1883, 0.8366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7233, 0.3531]\", \"lift_point\": \"[0.7233, 0.3531]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7091, 0.4199]\", \"lift_point\": \"[0.7091, 0.4199]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.4719]\", \"lift_point\": \"[0.7956, 0.4719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1883, 0.8366]\", \"lift_point\": \"[0.1883, 0.8366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7233, 0.3531]\", \"lift_point\": \"[0.7233, 0.3531]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5377, 0.8055]\", \"lift_point\": \"[0.5377, 0.8055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7042, 0.4983]\", \"lift_point\": \"[0.7042, 0.4983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7214, 0.5225]\", \"lift_point\": \"[0.7214, 0.5225]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7956, 0.4719]\", \"lift_point\": \"[0.7956, 0.4719]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1883, 0.8366]\", \"lift_point\": \"[0.1883, 0.8366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7233, 0.3531]\", \"lift_point\": \"[0.7233, 0.3531]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5377, 0.8055]\", \"lift_point\": \"[0.5377, 0.8055]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65395_24.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5388, 0.6488]\", \"lift_point\": \"[0.5388, 0.6488]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8562, 0.4781]\", \"lift_point\": \"[0.8562, 0.4781]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6772, 0.7849]\", \"lift_point\": \"[0.6772, 0.7849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6772, 0.7849]\", \"lift_point\": \"[0.6772, 0.7849]\", \"typed_text\": \"\" Goal: Set an alarm for 7pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65790_29.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8943, 0.3763]\", \"lift_point\": \"[0.8943, 0.3763]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65804_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8943, 0.3763]\", \"lift_point\": \"[0.8943, 0.3763]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65804_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8943, 0.3763]\", \"lift_point\": \"[0.8943, 0.3763]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.5639]\", \"lift_point\": \"[0.1869, 0.5639]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65804_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8943, 0.3763]\", \"lift_point\": \"[0.8943, 0.3763]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.5639]\", \"lift_point\": \"[0.1869, 0.5639]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65804_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.8776]\", \"lift_point\": \"[0.7928, 0.8776]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.8776]\", \"lift_point\": \"[0.7928, 0.8776]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.7902]\", \"lift_point\": \"[0.0567, 0.7902]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.8776]\", \"lift_point\": \"[0.7928, 0.8776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.7902]\", \"lift_point\": \"[0.0567, 0.7902]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.9438]\", \"lift_point\": \"[0.0617, 0.9438]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.8776]\", \"lift_point\": \"[0.7928, 0.8776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.7902]\", \"lift_point\": \"[0.0567, 0.7902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.9438]\", \"lift_point\": \"[0.0617, 0.9438]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sms\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.8776]\", \"lift_point\": \"[0.7928, 0.8776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.7902]\", \"lift_point\": \"[0.0567, 0.7902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.9438]\", \"lift_point\": \"[0.0617, 0.9438]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sms\" Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1771, 0.5132]\", \"lift_point\": \"[0.1771, 0.5132]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7895, 0.3809]\", \"lift_point\": \"[0.7895, 0.3809]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.8776]\", \"lift_point\": \"[0.7928, 0.8776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.7902]\", \"lift_point\": \"[0.0567, 0.7902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.9438]\", \"lift_point\": \"[0.0617, 0.9438]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sms\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1771, 0.5132]\", \"lift_point\": \"[0.1771, 0.5132]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1501, 0.4031]\", \"lift_point\": \"[0.1501, 0.4031]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8945, 0.4723]\", \"lift_point\": \"[0.8945, 0.4723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9751, 0.4922]\", \"lift_point\": \"[0.9751, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.8776]\", \"lift_point\": \"[0.7928, 0.8776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0567, 0.7902]\", \"lift_point\": \"[0.0567, 0.7902]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.9438]\", \"lift_point\": \"[0.0617, 0.9438]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sms\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1771, 0.5132]\", \"lift_point\": \"[0.1771, 0.5132]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1501, 0.4031]\", \"lift_point\": \"[0.1501, 0.4031]\", \"typed_text\": \"\" Goal: Open the SMS app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65941_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" Goal: What time is it in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" Goal: What time is it in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.6024]\", \"lift_point\": \"[0.777, 0.6024]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.6024]\", \"lift_point\": \"[0.777, 0.6024]\", \"typed_text\": \"\" Goal: What time is it in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0511, 0.613]\", \"lift_point\": \"[0.0511, 0.613]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.6024]\", \"lift_point\": \"[0.777, 0.6024]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0511, 0.613]\", \"lift_point\": \"[0.0511, 0.613]\", \"typed_text\": \"\" Goal: What time is it in Paris?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.6024]\", \"lift_point\": \"[0.777, 0.6024]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0511, 0.613]\", \"lift_point\": \"[0.0511, 0.613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in paris\" Goal: What time is it in Paris?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.6024]\", \"lift_point\": \"[0.777, 0.6024]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0511, 0.613]\", \"lift_point\": \"[0.0511, 0.613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in Paris?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.3964]\", \"lift_point\": \"[0.1221, 0.3964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9793, 0.4828]\", \"lift_point\": \"[0.9793, 0.4828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.777, 0.6024]\", \"lift_point\": \"[0.777, 0.6024]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0511, 0.613]\", \"lift_point\": \"[0.0511, 0.613]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in paris\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1221, 0.3964]\", \"lift_point\": \"[0.1221, 0.3964]\", \"typed_text\": \"\" Goal: What time is it in Paris?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/65967_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.3437]\", \"lift_point\": \"[0.0596, 0.3437]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.3437]\", \"lift_point\": \"[0.0596, 0.3437]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.3437]\", \"lift_point\": \"[0.0596, 0.3437]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1124, 0.5747]\", \"lift_point\": \"[0.1124, 0.5747]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.3437]\", \"lift_point\": \"[0.0596, 0.3437]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1124, 0.5747]\", \"lift_point\": \"[0.1124, 0.5747]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2681, 0.7456]\", \"lift_point\": \"[0.2681, 0.7456]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.3437]\", \"lift_point\": \"[0.0596, 0.3437]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1124, 0.5747]\", \"lift_point\": \"[0.1124, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2681, 0.7456]\", \"lift_point\": \"[0.2681, 0.7456]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.3437]\", \"lift_point\": \"[0.0596, 0.3437]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1124, 0.5747]\", \"lift_point\": \"[0.1124, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2681, 0.7456]\", \"lift_point\": \"[0.2681, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.464]\", \"lift_point\": \"[0.6098, 0.464]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9834, 0.4958]\", \"lift_point\": \"[0.9834, 0.4958]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.6124]\", \"lift_point\": \"[0.7979, 0.6124]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.3437]\", \"lift_point\": \"[0.0596, 0.3437]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from Helsinki to Seoul\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1124, 0.5747]\", \"lift_point\": \"[0.1124, 0.5747]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2681, 0.7456]\", \"lift_point\": \"[0.2681, 0.7456]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6098, 0.464]\", \"lift_point\": \"[0.6098, 0.464]\", \"typed_text\": \"\" Goal: Search for flights from Helsinki to Seoul>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66087_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.8487]\", \"lift_point\": \"[0.7914, 0.8487]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.8487]\", \"lift_point\": \"[0.7914, 0.8487]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4713]\", \"lift_point\": \"[0.8975, 0.4713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.8487]\", \"lift_point\": \"[0.7914, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4713]\", \"lift_point\": \"[0.8975, 0.4713]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3378, 0.7459]\", \"lift_point\": \"[0.3378, 0.7459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.8487]\", \"lift_point\": \"[0.7914, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4713]\", \"lift_point\": \"[0.8975, 0.4713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3378, 0.7459]\", \"lift_point\": \"[0.3378, 0.7459]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5648, 0.6812]\", \"lift_point\": \"[0.5648, 0.6812]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.8487]\", \"lift_point\": \"[0.7914, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4713]\", \"lift_point\": \"[0.8975, 0.4713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3378, 0.7459]\", \"lift_point\": \"[0.3378, 0.7459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5648, 0.6812]\", \"lift_point\": \"[0.5648, 0.6812]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4274, 0.5059]\", \"lift_point\": \"[0.4274, 0.5059]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.8487]\", \"lift_point\": \"[0.7914, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4713]\", \"lift_point\": \"[0.8975, 0.4713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3378, 0.7459]\", \"lift_point\": \"[0.3378, 0.7459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5648, 0.6812]\", \"lift_point\": \"[0.5648, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4274, 0.5059]\", \"lift_point\": \"[0.4274, 0.5059]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.7445]\", \"lift_point\": \"[0.6778, 0.7445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7283, 0.5746]\", \"lift_point\": \"[0.7283, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7765, 0.8598]\", \"lift_point\": \"[0.7765, 0.8598]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7914, 0.8487]\", \"lift_point\": \"[0.7914, 0.8487]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8975, 0.4713]\", \"lift_point\": \"[0.8975, 0.4713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3378, 0.7459]\", \"lift_point\": \"[0.3378, 0.7459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5648, 0.6812]\", \"lift_point\": \"[0.5648, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4274, 0.5059]\", \"lift_point\": \"[0.4274, 0.5059]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.7445]\", \"lift_point\": \"[0.6778, 0.7445]\", \"typed_text\": \"\" Goal: Set an alarm for 4pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66218_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of England?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.6121]\", \"lift_point\": \"[0.7942, 0.6121]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66309_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.6121]\", \"lift_point\": \"[0.7942, 0.6121]\", \"typed_text\": \"\" Goal: What is the capital of England?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.4296]\", \"lift_point\": \"[0.0553, 0.4296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66309_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.6121]\", \"lift_point\": \"[0.7942, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.4296]\", \"lift_point\": \"[0.0553, 0.4296]\", \"typed_text\": \"\" Goal: What is the capital of England?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of England\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66309_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.6121]\", \"lift_point\": \"[0.7942, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.4296]\", \"lift_point\": \"[0.0553, 0.4296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of England\" Goal: What is the capital of England?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66309_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7942, 0.6121]\", \"lift_point\": \"[0.7942, 0.6121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.4296]\", \"lift_point\": \"[0.0553, 0.4296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of England\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of England?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66309_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7805, 0.6321]\", \"lift_point\": \"[0.7805, 0.6321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.065, 0.495]\", \"lift_point\": \"[0.065, 0.495]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"pizza Hut\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1174, 0.3098]\", \"lift_point\": \"[0.1174, 0.3098]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.425, 0.1405]\", \"lift_point\": \"[0.425, 0.1405]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6194, 0.1229]\", \"lift_point\": \"[0.6194, 0.1229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1167, 0.6356]\", \"lift_point\": \"[0.1167, 0.6356]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2708, 0.4597]\", \"lift_point\": \"[0.2708, 0.4597]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1167, 0.6356]\", \"lift_point\": \"[0.1167, 0.6356]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5038, 0.4206]\", \"lift_point\": \"[0.5038, 0.4206]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0703, 0.0858]\", \"lift_point\": \"[0.0703, 0.0858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1167, 0.6356]\", \"lift_point\": \"[0.1167, 0.6356]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5038, 0.4206]\", \"lift_point\": \"[0.5038, 0.4206]\", \"typed_text\": \"\" Goal: What's on the menu at Pizza Hut?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66341_25.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.5151]\", \"lift_point\": \"[0.059, 0.5151]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.5151]\", \"lift_point\": \"[0.059, 0.5151]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.5151]\", \"lift_point\": \"[0.059, 0.5151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.5151]\", \"lift_point\": \"[0.059, 0.5151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.5151]\", \"lift_point\": \"[0.059, 0.5151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6927, 0.5764]\", \"lift_point\": \"[0.6927, 0.5764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6395, 0.8607]\", \"lift_point\": \"[0.6395, 0.8607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.5151]\", \"lift_point\": \"[0.059, 0.5151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6927, 0.5764]\", \"lift_point\": \"[0.6927, 0.5764]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1692, 0.4973]\", \"lift_point\": \"[0.1692, 0.4973]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.0814]\", \"lift_point\": \"[0.0551, 0.0814]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.052, 0.5992]\", \"lift_point\": \"[0.052, 0.5992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.059, 0.5151]\", \"lift_point\": \"[0.059, 0.5151]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6927, 0.5764]\", \"lift_point\": \"[0.6927, 0.5764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1692, 0.4973]\", \"lift_point\": \"[0.1692, 0.4973]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66452_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6017]\", \"lift_point\": \"[0.0584, 0.6017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6017]\", \"lift_point\": \"[0.0584, 0.6017]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6017]\", \"lift_point\": \"[0.0584, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Mexico city\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6017]\", \"lift_point\": \"[0.0584, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.3211]\", \"lift_point\": \"[0.1154, 0.3211]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6017]\", \"lift_point\": \"[0.0584, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.3211]\", \"lift_point\": \"[0.1154, 0.3211]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6017]\", \"lift_point\": \"[0.0584, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.3211]\", \"lift_point\": \"[0.1154, 0.3211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6861, 0.5225]\", \"lift_point\": \"[0.6861, 0.5225]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.6339]\", \"lift_point\": \"[0.8049, 0.6339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6017]\", \"lift_point\": \"[0.0584, 0.6017]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1154, 0.3211]\", \"lift_point\": \"[0.1154, 0.3211]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6861, 0.5225]\", \"lift_point\": \"[0.6861, 0.5225]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66486_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a bicycle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9043, 0.4951]\", \"lift_point\": \"[0.9043, 0.4951]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66578_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9043, 0.4951]\", \"lift_point\": \"[0.9043, 0.4951]\", \"typed_text\": \"\" Goal: What is the speed of a bicycle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a bicycle?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66578_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9043, 0.4951]\", \"lift_point\": \"[0.9043, 0.4951]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a bicycle?\" Goal: What is the speed of a bicycle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4848]\", \"lift_point\": \"[0.8837, 0.4848]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66578_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9043, 0.4951]\", \"lift_point\": \"[0.9043, 0.4951]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a bicycle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4848]\", \"lift_point\": \"[0.8837, 0.4848]\", \"typed_text\": \"\" Goal: What is the speed of a bicycle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.6036]\", \"lift_point\": \"[0.1282, 0.6036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66578_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9043, 0.4951]\", \"lift_point\": \"[0.9043, 0.4951]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a bicycle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4848]\", \"lift_point\": \"[0.8837, 0.4848]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.6036]\", \"lift_point\": \"[0.1282, 0.6036]\", \"typed_text\": \"\" Goal: What is the speed of a bicycle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4073, 0.5385]\", \"lift_point\": \"[0.4073, 0.5385]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66578_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9043, 0.4951]\", \"lift_point\": \"[0.9043, 0.4951]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a bicycle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8837, 0.4848]\", \"lift_point\": \"[0.8837, 0.4848]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.6036]\", \"lift_point\": \"[0.1282, 0.6036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4073, 0.5385]\", \"lift_point\": \"[0.4073, 0.5385]\", \"typed_text\": \"\" Goal: What is the speed of a bicycle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66578_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8596, 0.5593]\", \"lift_point\": \"[0.8596, 0.5593]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66728_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8596, 0.5593]\", \"lift_point\": \"[0.8596, 0.5593]\", \"typed_text\": \"\" Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a skateboard?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66728_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8596, 0.5593]\", \"lift_point\": \"[0.8596, 0.5593]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a skateboard?\" Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.4947]\", \"lift_point\": \"[0.1142, 0.4947]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66728_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8596, 0.5593]\", \"lift_point\": \"[0.8596, 0.5593]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a skateboard?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.4947]\", \"lift_point\": \"[0.1142, 0.4947]\", \"typed_text\": \"\" Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66728_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4912]\", \"lift_point\": \"[0.8947, 0.4912]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66732_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4912]\", \"lift_point\": \"[0.8947, 0.4912]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Target?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66732_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4912]\", \"lift_point\": \"[0.8947, 0.4912]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Target?\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1275, 0.294]\", \"lift_point\": \"[0.1275, 0.294]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66732_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8947, 0.4912]\", \"lift_point\": \"[0.8947, 0.4912]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1275, 0.294]\", \"lift_point\": \"[0.1275, 0.294]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66732_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.888]\", \"lift_point\": \"[0.783, 0.888]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66752_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.888]\", \"lift_point\": \"[0.783, 0.888]\", \"typed_text\": \"\" Goal: Turn on notifications for the Google Maps app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66752_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new mascara>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66800_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8718, 0.3458]\", \"lift_point\": \"[0.8718, 0.3458]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66800_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8718, 0.3458]\", \"lift_point\": \"[0.8718, 0.3458]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new mascara\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66800_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8718, 0.3458]\", \"lift_point\": \"[0.8718, 0.3458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new mascara\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1328, 0.2844]\", \"lift_point\": \"[0.1328, 0.2844]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66800_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8718, 0.3458]\", \"lift_point\": \"[0.8718, 0.3458]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for a new mascara\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1328, 0.2844]\", \"lift_point\": \"[0.1328, 0.2844]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66800_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.6234]\", \"lift_point\": \"[0.8819, 0.6234]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66928_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.6234]\", \"lift_point\": \"[0.8819, 0.6234]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66928_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.6234]\", \"lift_point\": \"[0.8819, 0.6234]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1206, 0.338]\", \"lift_point\": \"[0.1206, 0.338]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66928_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.6234]\", \"lift_point\": \"[0.8819, 0.6234]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1206, 0.338]\", \"lift_point\": \"[0.1206, 0.338]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6716, 0.519]\", \"lift_point\": \"[0.6716, 0.519]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66928_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.6234]\", \"lift_point\": \"[0.8819, 0.6234]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1206, 0.338]\", \"lift_point\": \"[0.1206, 0.338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6716, 0.519]\", \"lift_point\": \"[0.6716, 0.519]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4584, 0.5643]\", \"lift_point\": \"[0.4584, 0.5643]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66928_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8819, 0.6234]\", \"lift_point\": \"[0.8819, 0.6234]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Trinidad and Tobago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1206, 0.338]\", \"lift_point\": \"[0.1206, 0.338]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6716, 0.519]\", \"lift_point\": \"[0.6716, 0.519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4584, 0.5643]\", \"lift_point\": \"[0.4584, 0.5643]\", \"typed_text\": \"\" Goal: What's the news in Trinidad and Tobago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66928_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new hair product>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8795, 0.3776]\", \"lift_point\": \"[0.8795, 0.3776]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8795, 0.3776]\", \"lift_point\": \"[0.8795, 0.3776]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new hair product\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8795, 0.3776]\", \"lift_point\": \"[0.8795, 0.3776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new hair product\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.3894]\", \"lift_point\": \"[0.12, 0.3894]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8795, 0.3776]\", \"lift_point\": \"[0.8795, 0.3776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new hair product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.3894]\", \"lift_point\": \"[0.12, 0.3894]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8795, 0.3776]\", \"lift_point\": \"[0.8795, 0.3776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new hair product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.3894]\", \"lift_point\": \"[0.12, 0.3894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7147, 0.4652]\", \"lift_point\": \"[0.7147, 0.4652]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8795, 0.3776]\", \"lift_point\": \"[0.8795, 0.3776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new hair product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.3894]\", \"lift_point\": \"[0.12, 0.3894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7147, 0.4652]\", \"lift_point\": \"[0.7147, 0.4652]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5253]\", \"lift_point\": \"[0.8376, 0.5253]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8795, 0.3776]\", \"lift_point\": \"[0.8795, 0.3776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new hair product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.12, 0.3894]\", \"lift_point\": \"[0.12, 0.3894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7147, 0.4652]\", \"lift_point\": \"[0.7147, 0.4652]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5253]\", \"lift_point\": \"[0.8376, 0.5253]\", \"typed_text\": \"\" Goal: Search for a new hair product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/66995_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8851, 0.4398]\", \"lift_point\": \"[0.8851, 0.4398]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67071_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8851, 0.4398]\", \"lift_point\": \"[0.8851, 0.4398]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest T mobile store?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67071_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8851, 0.4398]\", \"lift_point\": \"[0.8851, 0.4398]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest T mobile store?\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0499, 0.3775]\", \"lift_point\": \"[0.0499, 0.3775]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67071_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8851, 0.4398]\", \"lift_point\": \"[0.8851, 0.4398]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest T mobile store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0499, 0.3775]\", \"lift_point\": \"[0.0499, 0.3775]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.6011]\", \"lift_point\": \"[0.1241, 0.6011]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67071_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8851, 0.4398]\", \"lift_point\": \"[0.8851, 0.4398]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest T mobile store?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0499, 0.3775]\", \"lift_point\": \"[0.0499, 0.3775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1241, 0.6011]\", \"lift_point\": \"[0.1241, 0.6011]\", \"typed_text\": \"\" Goal: How do I get to the nearest T-Mobile Store?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67071_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8929, 0.491]\", \"lift_point\": \"[0.8929, 0.491]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.118, 0.5821]\", \"lift_point\": \"[0.118, 0.5821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.4995]\", \"lift_point\": \"[0.3566, 0.4995]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4245, 0.3739]\", \"lift_point\": \"[0.4245, 0.3739]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8862, 0.5515]\", \"lift_point\": \"[0.8862, 0.5515]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.4995]\", \"lift_point\": \"[0.3566, 0.4995]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67119_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7923, 0.698]\", \"lift_point\": \"[0.7923, 0.698]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.6342]\", \"lift_point\": \"[0.0604, 0.6342]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.923]\", \"lift_point\": \"[0.0633, 0.923]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"papa murphys\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.4987]\", \"lift_point\": \"[0.2559, 0.4987]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.4987]\", \"lift_point\": \"[0.2559, 0.4987]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1462, 0.4765]\", \"lift_point\": \"[0.1462, 0.4765]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.441, 0.3817]\", \"lift_point\": \"[0.441, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2559, 0.4987]\", \"lift_point\": \"[0.2559, 0.4987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1462, 0.4765]\", \"lift_point\": \"[0.1462, 0.4765]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67207_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6762]\", \"lift_point\": \"[0.7832, 0.6762]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.785, 0.6966]\", \"lift_point\": \"[0.785, 0.6966]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7786, 0.6947]\", \"lift_point\": \"[0.7786, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [TYPE,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6731]\", \"lift_point\": \"[0.7752, 0.6731]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7832, 0.6699]\", \"lift_point\": \"[0.7832, 0.6699]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.682]\", \"lift_point\": \"[0.0578, 0.682]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bed room apartment rent in houston\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"in houston\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.194, 0.4928]\", \"lift_point\": \"[0.194, 0.4928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6289, 0.3562]\", \"lift_point\": \"[0.6289, 0.3562]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2735, 0.9223]\", \"lift_point\": \"[0.2735, 0.9223]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1932, 0.5899]\", \"lift_point\": \"[0.1932, 0.5899]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3812, 0.4667]\", \"lift_point\": \"[0.3812, 0.4667]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1932, 0.5899]\", \"lift_point\": \"[0.1932, 0.5899]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Houston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67689_42.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7887, 0.6584]\", \"lift_point\": \"[0.7887, 0.6584]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8068, 0.681]\", \"lift_point\": \"[0.8068, 0.681]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7864, 0.6789]\", \"lift_point\": \"[0.7864, 0.6789]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7822, 0.7046]\", \"lift_point\": \"[0.7822, 0.7046]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8035, 0.6608]\", \"lift_point\": \"[0.8035, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8021, 0.6608]\", \"lift_point\": \"[0.8021, 0.6608]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7996, 0.6634]\", \"lift_point\": \"[0.7996, 0.6634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0579, 0.4106]\", \"lift_point\": \"[0.0579, 0.4106]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7044, 0.409]\", \"lift_point\": \"[0.7044, 0.409]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good resturant near me\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2055, 0.3872]\", \"lift_point\": \"[0.2055, 0.3872]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7044, 0.409]\", \"lift_point\": \"[0.7044, 0.409]\", \"typed_text\": \"\" Goal: What's a good restaurant near me?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/67924_32.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7842, 0.6959]\", \"lift_point\": \"[0.7842, 0.6959]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.4778]\", \"lift_point\": \"[0.0606, 0.4778]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"whats the news in taiwan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2049, 0.3624]\", \"lift_point\": \"[0.2049, 0.3624]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7282, 0.2723]\", \"lift_point\": \"[0.7282, 0.2723]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.8299]\", \"lift_point\": \"[0.7663, 0.8299]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7663, 0.8299]\", \"lift_point\": \"[0.7663, 0.8299]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68010_15.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7088]\", \"lift_point\": \"[0.7884, 0.7088]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8173, 0.6623]\", \"lift_point\": \"[0.8173, 0.6623]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6821]\", \"lift_point\": \"[0.7794, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6933]\", \"lift_point\": \"[0.7794, 0.6933]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8112, 0.675]\", \"lift_point\": \"[0.8112, 0.675]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.81, 0.6775]\", \"lift_point\": \"[0.81, 0.6775]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0635, 0.5123]\", \"lift_point\": \"[0.0635, 0.5123]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flight from zurich to buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2101, 0.4208]\", \"lift_point\": \"[0.2101, 0.4208]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8827, 0.205]\", \"lift_point\": \"[0.8827, 0.205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6941, 0.5458]\", \"lift_point\": \"[0.6941, 0.5458]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3321, 0.1299]\", \"lift_point\": \"[0.3321, 0.1299]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.6403]\", \"lift_point\": \"[0.3076, 0.6403]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.933]\", \"lift_point\": \"[0.1229, 0.933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.6403]\", \"lift_point\": \"[0.3076, 0.6403]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1304, 0.3286]\", \"lift_point\": \"[0.1304, 0.3286]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4991, 0.8948]\", \"lift_point\": \"[0.4991, 0.8948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.6403]\", \"lift_point\": \"[0.3076, 0.6403]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1304, 0.3286]\", \"lift_point\": \"[0.1304, 0.3286]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9007, 0.396]\", \"lift_point\": \"[0.9007, 0.396]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.402, 0.9084]\", \"lift_point\": \"[0.402, 0.9084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.6403]\", \"lift_point\": \"[0.3076, 0.6403]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1304, 0.3286]\", \"lift_point\": \"[0.1304, 0.3286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9007, 0.396]\", \"lift_point\": \"[0.9007, 0.396]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.5247]\", \"lift_point\": \"[0.5737, 0.5247]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.5131]\", \"lift_point\": \"[0.8984, 0.5131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.6403]\", \"lift_point\": \"[0.3076, 0.6403]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1304, 0.3286]\", \"lift_point\": \"[0.1304, 0.3286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9007, 0.396]\", \"lift_point\": \"[0.9007, 0.396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.5247]\", \"lift_point\": \"[0.5737, 0.5247]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.4744]\", \"lift_point\": \"[0.6477, 0.4744]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6419, 0.9862]\", \"lift_point\": \"[0.6419, 0.9862]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.6403]\", \"lift_point\": \"[0.3076, 0.6403]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1304, 0.3286]\", \"lift_point\": \"[0.1304, 0.3286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9007, 0.396]\", \"lift_point\": \"[0.9007, 0.396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.5247]\", \"lift_point\": \"[0.5737, 0.5247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.4744]\", \"lift_point\": \"[0.6477, 0.4744]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.7882]\", \"lift_point\": \"[0.5543, 0.7882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5557, 0.4411]\", \"lift_point\": \"[0.5557, 0.4411]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4457, 0.1982]\", \"lift_point\": \"[0.4457, 0.1982]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3076, 0.6403]\", \"lift_point\": \"[0.3076, 0.6403]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1304, 0.3286]\", \"lift_point\": \"[0.1304, 0.3286]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9007, 0.396]\", \"lift_point\": \"[0.9007, 0.396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5737, 0.5247]\", \"lift_point\": \"[0.5737, 0.5247]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6477, 0.4744]\", \"lift_point\": \"[0.6477, 0.4744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5543, 0.7882]\", \"lift_point\": \"[0.5543, 0.7882]\", \"typed_text\": \"\" Goal: Search for flights from Zurich to Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68164_32.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7823, 0.7307]\", \"lift_point\": \"[0.7823, 0.7307]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7823, 0.7307]\", \"lift_point\": \"[0.7823, 0.7307]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7708, 0.7363]\", \"lift_point\": \"[0.7708, 0.7363]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7823, 0.7307]\", \"lift_point\": \"[0.7823, 0.7307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7708, 0.7363]\", \"lift_point\": \"[0.7708, 0.7363]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.71, 0.3772]\", \"lift_point\": \"[0.71, 0.3772]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7823, 0.7307]\", \"lift_point\": \"[0.7823, 0.7307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7708, 0.7363]\", \"lift_point\": \"[0.7708, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.71, 0.3772]\", \"lift_point\": \"[0.71, 0.3772]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6367, 0.3181]\", \"lift_point\": \"[0.6367, 0.3181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7823, 0.7307]\", \"lift_point\": \"[0.7823, 0.7307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7708, 0.7363]\", \"lift_point\": \"[0.7708, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.71, 0.3772]\", \"lift_point\": \"[0.71, 0.3772]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6367, 0.3181]\", \"lift_point\": \"[0.6367, 0.3181]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.9241]\", \"lift_point\": \"[0.0624, 0.9241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7823, 0.7307]\", \"lift_point\": \"[0.7823, 0.7307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7708, 0.7363]\", \"lift_point\": \"[0.7708, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.71, 0.3772]\", \"lift_point\": \"[0.71, 0.3772]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6367, 0.3181]\", \"lift_point\": \"[0.6367, 0.3181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.9241]\", \"lift_point\": \"[0.0624, 0.9241]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1249, 0.4824]\", \"lift_point\": \"[0.1249, 0.4824]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7823, 0.7307]\", \"lift_point\": \"[0.7823, 0.7307]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7708, 0.7363]\", \"lift_point\": \"[0.7708, 0.7363]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.71, 0.3772]\", \"lift_point\": \"[0.71, 0.3772]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6367, 0.3181]\", \"lift_point\": \"[0.6367, 0.3181]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0624, 0.9241]\", \"lift_point\": \"[0.0624, 0.9241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1249, 0.4824]\", \"lift_point\": \"[0.1249, 0.4824]\", \"typed_text\": \"\" Goal: Open a new Chrome tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68175_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8727, 0.6751]\", \"lift_point\": \"[0.8727, 0.6751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8831, 0.3427]\", \"lift_point\": \"[0.8831, 0.3427]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the news in Guyana\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1568, 0.4054]\", \"lift_point\": \"[0.1568, 0.4054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6651, 0.9635]\", \"lift_point\": \"[0.6651, 0.9635]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6651, 0.9635]\", \"lift_point\": \"[0.6651, 0.9635]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6651, 0.9635]\", \"lift_point\": \"[0.6651, 0.9635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7261, 0.2077]\", \"lift_point\": \"[0.7261, 0.2077]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6651, 0.9635]\", \"lift_point\": \"[0.6651, 0.9635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6651, 0.9635]\", \"lift_point\": \"[0.6651, 0.9635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5494, 0.2482]\", \"lift_point\": \"[0.5494, 0.2482]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6651, 0.9635]\", \"lift_point\": \"[0.6651, 0.9635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3294, 0.0331]\", \"lift_point\": \"[0.3294, 0.0331]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6651, 0.9635]\", \"lift_point\": \"[0.6651, 0.9635]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3294, 0.0331]\", \"lift_point\": \"[0.3294, 0.0331]\", \"typed_text\": \"\" Goal: What's the news in Guyana?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68758_21.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8671, 0.5147]\", \"lift_point\": \"[0.8671, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8686, 0.5147]\", \"lift_point\": \"[0.8686, 0.5147]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the capital of Germany\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1642, 0.3681]\", \"lift_point\": \"[0.1642, 0.3681]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.713, 0.7036]\", \"lift_point\": \"[0.713, 0.7036]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5715, 0.8273]\", \"lift_point\": \"[0.5715, 0.8273]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5038]\", \"lift_point\": \"[0.8949, 0.5038]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5715, 0.8273]\", \"lift_point\": \"[0.5715, 0.8273]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68813_22.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8766, 0.4703]\", \"lift_point\": \"[0.8766, 0.4703]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8766, 0.4703]\", \"lift_point\": \"[0.8766, 0.4703]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8745, 0.1695]\", \"lift_point\": \"[0.8745, 0.1695]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8766, 0.4703]\", \"lift_point\": \"[0.8766, 0.4703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8745, 0.1695]\", \"lift_point\": \"[0.8745, 0.1695]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.382, 0.3512]\", \"lift_point\": \"[0.382, 0.3512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8766, 0.4703]\", \"lift_point\": \"[0.8766, 0.4703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8745, 0.1695]\", \"lift_point\": \"[0.8745, 0.1695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.382, 0.3512]\", \"lift_point\": \"[0.382, 0.3512]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of panama\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8766, 0.4703]\", \"lift_point\": \"[0.8766, 0.4703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8745, 0.1695]\", \"lift_point\": \"[0.8745, 0.1695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.382, 0.3512]\", \"lift_point\": \"[0.382, 0.3512]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of panama\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1539, 0.2404]\", \"lift_point\": \"[0.1539, 0.2404]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8766, 0.4703]\", \"lift_point\": \"[0.8766, 0.4703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8745, 0.1695]\", \"lift_point\": \"[0.8745, 0.1695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.382, 0.3512]\", \"lift_point\": \"[0.382, 0.3512]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of panama\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1539, 0.2404]\", \"lift_point\": \"[0.1539, 0.2404]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8306, 0.9188]\", \"lift_point\": \"[0.8306, 0.9188]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7467, 0.3167]\", \"lift_point\": \"[0.7467, 0.3167]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8766, 0.4703]\", \"lift_point\": \"[0.8766, 0.4703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8745, 0.1695]\", \"lift_point\": \"[0.8745, 0.1695]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.382, 0.3512]\", \"lift_point\": \"[0.382, 0.3512]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of panama\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1539, 0.2404]\", \"lift_point\": \"[0.1539, 0.2404]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8306, 0.9188]\", \"lift_point\": \"[0.8306, 0.9188]\", \"typed_text\": \"\" Goal: Google the capital of Panama>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/68898_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the camera>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69113_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69113_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3785, 0.3078]\", \"lift_point\": \"[0.3785, 0.3078]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69113_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3785, 0.3078]\", \"lift_point\": \"[0.3785, 0.3078]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8673, 0.497]\", \"lift_point\": \"[0.8673, 0.497]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69113_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3785, 0.3078]\", \"lift_point\": \"[0.3785, 0.3078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8673, 0.497]\", \"lift_point\": \"[0.8673, 0.497]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4783, 0.4587]\", \"lift_point\": \"[0.4783, 0.4587]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69113_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3785, 0.3078]\", \"lift_point\": \"[0.3785, 0.3078]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8673, 0.497]\", \"lift_point\": \"[0.8673, 0.497]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4783, 0.4587]\", \"lift_point\": \"[0.4783, 0.4587]\", \"typed_text\": \"\" Goal: Open the camera>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69113_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7515, 0.3131]\", \"lift_point\": \"[0.7515, 0.3131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69266_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7515, 0.3131]\", \"lift_point\": \"[0.7515, 0.3131]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.3344]\", \"lift_point\": \"[0.0593, 0.3344]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69266_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7515, 0.3131]\", \"lift_point\": \"[0.7515, 0.3131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.3344]\", \"lift_point\": \"[0.0593, 0.3344]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Austin?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69266_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7515, 0.3131]\", \"lift_point\": \"[0.7515, 0.3131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.3344]\", \"lift_point\": \"[0.0593, 0.3344]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Austin?\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1829, 0.4346]\", \"lift_point\": \"[0.1829, 0.4346]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69266_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7515, 0.3131]\", \"lift_point\": \"[0.7515, 0.3131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.3344]\", \"lift_point\": \"[0.0593, 0.3344]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Austin?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1829, 0.4346]\", \"lift_point\": \"[0.1829, 0.4346]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4635, 0.2316]\", \"lift_point\": \"[0.4635, 0.2316]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69266_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7515, 0.3131]\", \"lift_point\": \"[0.7515, 0.3131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0593, 0.3344]\", \"lift_point\": \"[0.0593, 0.3344]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Austin?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1829, 0.4346]\", \"lift_point\": \"[0.1829, 0.4346]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4635, 0.2316]\", \"lift_point\": \"[0.4635, 0.2316]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69266_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Starbucks?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Starbucks?\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1558, 0.2755]\", \"lift_point\": \"[0.1558, 0.2755]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Starbucks?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1558, 0.2755]\", \"lift_point\": \"[0.1558, 0.2755]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4626, 0.2054]\", \"lift_point\": \"[0.4626, 0.2054]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Starbucks?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1558, 0.2755]\", \"lift_point\": \"[0.1558, 0.2755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4626, 0.2054]\", \"lift_point\": \"[0.4626, 0.2054]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.571, 0.5377]\", \"lift_point\": \"[0.571, 0.5377]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Starbucks?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1558, 0.2755]\", \"lift_point\": \"[0.1558, 0.2755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4626, 0.2054]\", \"lift_point\": \"[0.4626, 0.2054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.571, 0.5377]\", \"lift_point\": \"[0.571, 0.5377]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5296, 0.4893]\", \"lift_point\": \"[0.5296, 0.4893]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7529, 0.3197]\", \"lift_point\": \"[0.7529, 0.3197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3899, 0.2991]\", \"lift_point\": \"[0.3899, 0.2991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Starbucks?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1558, 0.2755]\", \"lift_point\": \"[0.1558, 0.2755]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4626, 0.2054]\", \"lift_point\": \"[0.4626, 0.2054]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.571, 0.5377]\", \"lift_point\": \"[0.571, 0.5377]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5296, 0.4893]\", \"lift_point\": \"[0.5296, 0.4893]\", \"typed_text\": \"\" Goal: What's on the menu at Starbucks?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69428_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.74, 0.3022]\", \"lift_point\": \"[0.74, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0709, 0.5274]\", \"lift_point\": \"[0.0709, 0.5274]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" news in Malaysia?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.173, 0.2474]\", \"lift_point\": \"[0.173, 0.2474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4484, 0.1824]\", \"lift_point\": \"[0.4484, 0.1824]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5666, 0.5369]\", \"lift_point\": \"[0.5666, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4535, 0.855]\", \"lift_point\": \"[0.4535, 0.855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5201, 0.0532]\", \"lift_point\": \"[0.5201, 0.0532]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5201, 0.0532]\", \"lift_point\": \"[0.5201, 0.0532]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69485_24.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7229, 0.6176]\", \"lift_point\": \"[0.7229, 0.6176]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69645_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7229, 0.6176]\", \"lift_point\": \"[0.7229, 0.6176]\", \"typed_text\": \"\" Goal: Open a new tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0977, 0.916]\", \"lift_point\": \"[0.0977, 0.916]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69645_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7229, 0.6176]\", \"lift_point\": \"[0.7229, 0.6176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0977, 0.916]\", \"lift_point\": \"[0.0977, 0.916]\", \"typed_text\": \"\" Goal: Open a new tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1841, 0.3569]\", \"lift_point\": \"[0.1841, 0.3569]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69645_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7229, 0.6176]\", \"lift_point\": \"[0.7229, 0.6176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0977, 0.916]\", \"lift_point\": \"[0.0977, 0.916]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1841, 0.3569]\", \"lift_point\": \"[0.1841, 0.3569]\", \"typed_text\": \"\" Goal: Open a new tab in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69645_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new incognito tab in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7173, 0.6105]\", \"lift_point\": \"[0.7173, 0.6105]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69649_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7173, 0.6105]\", \"lift_point\": \"[0.7173, 0.6105]\", \"typed_text\": \"\" Goal: Open a new incognito tab in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0932, 0.9095]\", \"lift_point\": \"[0.0932, 0.9095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69649_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7173, 0.6105]\", \"lift_point\": \"[0.7173, 0.6105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0932, 0.9095]\", \"lift_point\": \"[0.0932, 0.9095]\", \"typed_text\": \"\" Goal: Open a new incognito tab in the chrome app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2589, 0.6734]\", \"lift_point\": \"[0.2589, 0.6734]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69649_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7173, 0.6105]\", \"lift_point\": \"[0.7173, 0.6105]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0932, 0.9095]\", \"lift_point\": \"[0.0932, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2589, 0.6734]\", \"lift_point\": \"[0.2589, 0.6734]\", \"typed_text\": \"\" Goal: Open a new incognito tab in the chrome app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69649_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69810_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2312, 0.5041]\", \"lift_point\": \"[0.2312, 0.5041]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69810_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2312, 0.5041]\", \"lift_point\": \"[0.2312, 0.5041]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5785, 0.6817]\", \"lift_point\": \"[0.5785, 0.6817]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69810_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2312, 0.5041]\", \"lift_point\": \"[0.2312, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5785, 0.6817]\", \"lift_point\": \"[0.5785, 0.6817]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4352, 0.3802]\", \"lift_point\": \"[0.4352, 0.3802]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69810_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2312, 0.5041]\", \"lift_point\": \"[0.2312, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5785, 0.6817]\", \"lift_point\": \"[0.5785, 0.6817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4352, 0.3802]\", \"lift_point\": \"[0.4352, 0.3802]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4282, 0.7558]\", \"lift_point\": \"[0.4282, 0.7558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69810_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2312, 0.5041]\", \"lift_point\": \"[0.2312, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5785, 0.6817]\", \"lift_point\": \"[0.5785, 0.6817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4352, 0.3802]\", \"lift_point\": \"[0.4352, 0.3802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4282, 0.7558]\", \"lift_point\": \"[0.4282, 0.7558]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.3537]\", \"lift_point\": \"[0.8049, 0.3537]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69810_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2312, 0.5041]\", \"lift_point\": \"[0.2312, 0.5041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5785, 0.6817]\", \"lift_point\": \"[0.5785, 0.6817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4352, 0.3802]\", \"lift_point\": \"[0.4352, 0.3802]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4282, 0.7558]\", \"lift_point\": \"[0.4282, 0.7558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8049, 0.3537]\", \"lift_point\": \"[0.8049, 0.3537]\", \"typed_text\": \"\" Goal: Open the gallery>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/69810_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.825]\", \"lift_point\": \"[0.0464, 0.825]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.825]\", \"lift_point\": \"[0.0464, 0.825]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.873, 0.3966]\", \"lift_point\": \"[0.873, 0.3966]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.825]\", \"lift_point\": \"[0.0464, 0.825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video \" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8782, 0.2261]\", \"lift_point\": \"[0.8782, 0.2261]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.825]\", \"lift_point\": \"[0.0464, 0.825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7111, 0.5029]\", \"lift_point\": \"[0.7111, 0.5029]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.825]\", \"lift_point\": \"[0.0464, 0.825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7111, 0.5029]\", \"lift_point\": \"[0.7111, 0.5029]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2329, 0.8893]\", \"lift_point\": \"[0.2329, 0.8893]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7463, 0.4818]\", \"lift_point\": \"[0.7463, 0.4818]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0464, 0.825]\", \"lift_point\": \"[0.0464, 0.825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new Ed Sheeran video \" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7111, 0.5029]\", \"lift_point\": \"[0.7111, 0.5029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2329, 0.8893]\", \"lift_point\": \"[0.2329, 0.8893]\", \"typed_text\": \"\" Goal: Play the new Ed Sheeran video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70026_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70086_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8801, 0.5536]\", \"lift_point\": \"[0.8801, 0.5536]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70086_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8801, 0.5536]\", \"lift_point\": \"[0.8801, 0.5536]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Italy\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70086_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8801, 0.5536]\", \"lift_point\": \"[0.8801, 0.5536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Italy\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70086_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8801, 0.5536]\", \"lift_point\": \"[0.8801, 0.5536]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Italy\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Italy?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70086_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70136_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5328]\", \"lift_point\": \"[0.8949, 0.5328]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70136_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5328]\", \"lift_point\": \"[0.8949, 0.5328]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70136_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5328]\", \"lift_point\": \"[0.8949, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70136_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5328]\", \"lift_point\": \"[0.8949, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8306, 0.4972]\", \"lift_point\": \"[0.8306, 0.4972]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70136_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.5328]\", \"lift_point\": \"[0.8949, 0.5328]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Francisco\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8306, 0.4972]\", \"lift_point\": \"[0.8306, 0.4972]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70136_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in the Dominican Republic?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70273_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in the Dominican Republic?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.531]\", \"lift_point\": \"[0.8785, 0.531]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70273_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.531]\", \"lift_point\": \"[0.8785, 0.531]\", \"typed_text\": \"\" Goal: What's the news in the Dominican Republic?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Dominican Republic\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70273_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.531]\", \"lift_point\": \"[0.8785, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Dominican Republic\" Goal: What's the news in the Dominican Republic?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70273_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.531]\", \"lift_point\": \"[0.8785, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Dominican Republic\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in the Dominican Republic?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70273_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.531]\", \"lift_point\": \"[0.8785, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Dominican Republic\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in the Dominican Republic?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5953, 0.6958]\", \"lift_point\": \"[0.5953, 0.6958]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70273_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8785, 0.531]\", \"lift_point\": \"[0.8785, 0.531]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in the Dominican Republic\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5953, 0.6958]\", \"lift_point\": \"[0.5953, 0.6958]\", \"typed_text\": \"\" Goal: What's the news in the Dominican Republic?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70273_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70286_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70286_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6087, 0.0939]\", \"lift_point\": \"[0.6087, 0.0939]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70286_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6087, 0.0939]\", \"lift_point\": \"[0.6087, 0.0939]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.3335]\", \"lift_point\": \"[0.0503, 0.3335]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70286_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6087, 0.0939]\", \"lift_point\": \"[0.6087, 0.0939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.3335]\", \"lift_point\": \"[0.0503, 0.3335]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"burger restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70286_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6087, 0.0939]\", \"lift_point\": \"[0.6087, 0.0939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.3335]\", \"lift_point\": \"[0.0503, 0.3335]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"burger restaurants\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.5462]\", \"lift_point\": \"[0.1229, 0.5462]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70286_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6087, 0.0939]\", \"lift_point\": \"[0.6087, 0.0939]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0503, 0.3335]\", \"lift_point\": \"[0.0503, 0.3335]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"burger restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1229, 0.5462]\", \"lift_point\": \"[0.1229, 0.5462]\", \"typed_text\": \"\" Goal: Search for the best burger restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70286_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70291_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.7402]\", \"lift_point\": \"[0.8868, 0.7402]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70291_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.7402]\", \"lift_point\": \"[0.8868, 0.7402]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70291_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.7402]\", \"lift_point\": \"[0.8868, 0.7402]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time\" Goal: What's the time?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70291_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8868, 0.7402]\", \"lift_point\": \"[0.8868, 0.7402]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70291_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.5241]\", \"lift_point\": \"[0.8923, 0.5241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.5241]\", \"lift_point\": \"[0.8923, 0.5241]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.4164]\", \"lift_point\": \"[0.8908, 0.4164]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.5241]\", \"lift_point\": \"[0.8923, 0.5241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.4164]\", \"lift_point\": \"[0.8908, 0.4164]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.5241]\", \"lift_point\": \"[0.8923, 0.5241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.4164]\", \"lift_point\": \"[0.8908, 0.4164]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.5241]\", \"lift_point\": \"[0.8923, 0.5241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.4164]\", \"lift_point\": \"[0.8908, 0.4164]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2978, 0.5272]\", \"lift_point\": \"[0.2978, 0.5272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.5241]\", \"lift_point\": \"[0.8923, 0.5241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.4164]\", \"lift_point\": \"[0.8908, 0.4164]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2978, 0.5272]\", \"lift_point\": \"[0.2978, 0.5272]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5445, 0.8611]\", \"lift_point\": \"[0.5445, 0.8611]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.5241]\", \"lift_point\": \"[0.8923, 0.5241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8908, 0.4164]\", \"lift_point\": \"[0.8908, 0.4164]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Samsung TV\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2978, 0.5272]\", \"lift_point\": \"[0.2978, 0.5272]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5445, 0.8611]\", \"lift_point\": \"[0.5445, 0.8611]\", \"typed_text\": \"\" Goal: What's the price of the Samsung TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70305_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70336_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8774, 0.4726]\", \"lift_point\": \"[0.8774, 0.4726]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70336_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8774, 0.4726]\", \"lift_point\": \"[0.8774, 0.4726]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Los Angeles\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70336_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8774, 0.4726]\", \"lift_point\": \"[0.8774, 0.4726]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Los Angeles\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70336_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8774, 0.4726]\", \"lift_point\": \"[0.8774, 0.4726]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Los Angeles\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2682, 0.4703]\", \"lift_point\": \"[0.2682, 0.4703]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70336_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8774, 0.4726]\", \"lift_point\": \"[0.8774, 0.4726]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Los Angeles\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2682, 0.4703]\", \"lift_point\": \"[0.2682, 0.4703]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Los Angeles?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70336_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70341_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5341]\", \"lift_point\": \"[0.8932, 0.5341]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70341_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5341]\", \"lift_point\": \"[0.8932, 0.5341]\", \"typed_text\": \"\" Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a skateboard\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70341_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5341]\", \"lift_point\": \"[0.8932, 0.5341]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a skateboard\" Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70341_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5341]\", \"lift_point\": \"[0.8932, 0.5341]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a skateboard\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a skateboard?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70341_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6389]\", \"lift_point\": \"[0.7715, 0.6389]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6389]\", \"lift_point\": \"[0.7715, 0.6389]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.829, 0.6599]\", \"lift_point\": \"[0.829, 0.6599]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8934, 0.5381]\", \"lift_point\": \"[0.8934, 0.5381]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Greek restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7715, 0.6389]\", \"lift_point\": \"[0.7715, 0.6389]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.829, 0.6599]\", \"lift_point\": \"[0.829, 0.6599]\", \"typed_text\": \"\" Goal: Search for good Greek restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70404_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70480_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.5825]\", \"lift_point\": \"[0.8814, 0.5825]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70480_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.5825]\", \"lift_point\": \"[0.8814, 0.5825]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70480_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.5825]\", \"lift_point\": \"[0.8814, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70480_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.5825]\", \"lift_point\": \"[0.8814, 0.5825]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70480_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.1064]\", \"lift_point\": \"[0.5841, 0.1064]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.1064]\", \"lift_point\": \"[0.5841, 0.1064]\", \"typed_text\": \"\" Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.1064]\", \"lift_point\": \"[0.5841, 0.1064]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.5254]\", \"lift_point\": \"[0.0648, 0.5254]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.1064]\", \"lift_point\": \"[0.5841, 0.1064]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.5254]\", \"lift_point\": \"[0.0648, 0.5254]\", \"typed_text\": \"\" Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good pizza restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.1064]\", \"lift_point\": \"[0.5841, 0.1064]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.5254]\", \"lift_point\": \"[0.0648, 0.5254]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good pizza restaurants\" Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.5483]\", \"lift_point\": \"[0.1134, 0.5483]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5841, 0.1064]\", \"lift_point\": \"[0.5841, 0.1064]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0648, 0.5254]\", \"lift_point\": \"[0.0648, 0.5254]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good pizza restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1134, 0.5483]\", \"lift_point\": \"[0.1134, 0.5483]\", \"typed_text\": \"\" Goal: Search for good pizza restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70570_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70664_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.89, 0.5013]\", \"lift_point\": \"[0.89, 0.5013]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70664_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.89, 0.5013]\", \"lift_point\": \"[0.89, 0.5013]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Boston\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70664_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.89, 0.5013]\", \"lift_point\": \"[0.89, 0.5013]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Boston\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70664_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.89, 0.5013]\", \"lift_point\": \"[0.89, 0.5013]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Boston\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5569, 0.4846]\", \"lift_point\": \"[0.5569, 0.4846]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70664_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.89, 0.5013]\", \"lift_point\": \"[0.89, 0.5013]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Boston\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5569, 0.4846]\", \"lift_point\": \"[0.5569, 0.4846]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70664_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70670_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8931, 0.6537]\", \"lift_point\": \"[0.8931, 0.6537]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70670_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8931, 0.6537]\", \"lift_point\": \"[0.8931, 0.6537]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70670_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8931, 0.6537]\", \"lift_point\": \"[0.8931, 0.6537]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70670_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8931, 0.6537]\", \"lift_point\": \"[0.8931, 0.6537]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5998, 0.4981]\", \"lift_point\": \"[0.5998, 0.4981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70670_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8931, 0.6537]\", \"lift_point\": \"[0.8931, 0.6537]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my phone\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5998, 0.4981]\", \"lift_point\": \"[0.5998, 0.4981]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my phone>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70670_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8273, 0.9494]\", \"lift_point\": \"[0.8273, 0.9494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.886, 0.697]\", \"lift_point\": \"[0.886, 0.697]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8273, 0.9494]\", \"lift_point\": \"[0.8273, 0.9494]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8273, 0.9494]\", \"lift_point\": \"[0.8273, 0.9494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4201, 0.9891]\", \"lift_point\": \"[0.4201, 0.9891]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Indonesia\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2711, 0.4538]\", \"lift_point\": \"[0.2711, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8273, 0.9494]\", \"lift_point\": \"[0.8273, 0.9494]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4201, 0.9891]\", \"lift_point\": \"[0.4201, 0.9891]\", \"typed_text\": \"\" Goal: What's the news in Indonesia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70715_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Lyft app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.5055]\", \"lift_point\": \"[0.0481, 0.5055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.5055]\", \"lift_point\": \"[0.0481, 0.5055]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lyft app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.5055]\", \"lift_point\": \"[0.0481, 0.5055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lyft app\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.5055]\", \"lift_point\": \"[0.0481, 0.5055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lyft app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1345, 0.861]\", \"lift_point\": \"[0.1345, 0.861]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5894, 0.8664]\", \"lift_point\": \"[0.5894, 0.8664]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.5055]\", \"lift_point\": \"[0.0481, 0.5055]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lyft app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1345, 0.861]\", \"lift_point\": \"[0.1345, 0.861]\", \"typed_text\": \"\" Goal: Install the Lyft app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70744_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.892, 0.5796]\", \"lift_point\": \"[0.892, 0.5796]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70840_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.892, 0.5796]\", \"lift_point\": \"[0.892, 0.5796]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70840_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.892, 0.5796]\", \"lift_point\": \"[0.892, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70840_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.892, 0.5796]\", \"lift_point\": \"[0.892, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70840_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.892, 0.5796]\", \"lift_point\": \"[0.892, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.5803]\", \"lift_point\": \"[0.7979, 0.5803]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70840_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.892, 0.5796]\", \"lift_point\": \"[0.892, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.5803]\", \"lift_point\": \"[0.7979, 0.5803]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8996, 0.855]\", \"lift_point\": \"[0.8996, 0.855]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70840_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.892, 0.5796]\", \"lift_point\": \"[0.892, 0.5796]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Sri Lanka?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7979, 0.5803]\", \"lift_point\": \"[0.7979, 0.5803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8996, 0.855]\", \"lift_point\": \"[0.8996, 0.855]\", \"typed_text\": \"\" Goal: What's the news in Sri Lanka?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70840_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the settings>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70849_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70849_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7186, 0.3162]\", \"lift_point\": \"[0.7186, 0.3162]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70849_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7186, 0.3162]\", \"lift_point\": \"[0.7186, 0.3162]\", \"typed_text\": \"\" Goal: Open the settings>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70849_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the Google play store app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70897_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the Google play store app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70897_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the Google play store app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7219, 0.1204]\", \"lift_point\": \"[0.7219, 0.1204]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70897_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7219, 0.1204]\", \"lift_point\": \"[0.7219, 0.1204]\", \"typed_text\": \"\" Goal: Open the Google play store app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70897_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70951_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8911, 0.5494]\", \"lift_point\": \"[0.8911, 0.5494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70951_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8911, 0.5494]\", \"lift_point\": \"[0.8911, 0.5494]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70951_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8911, 0.5494]\", \"lift_point\": \"[0.8911, 0.5494]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70951_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8911, 0.5494]\", \"lift_point\": \"[0.8911, 0.5494]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/70951_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71177_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4997, 0.6738]\", \"lift_point\": \"[0.4997, 0.6738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71177_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4997, 0.6738]\", \"lift_point\": \"[0.4997, 0.6738]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.5631]\", \"lift_point\": \"[0.1282, 0.5631]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71177_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4997, 0.6738]\", \"lift_point\": \"[0.4997, 0.6738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.5631]\", \"lift_point\": \"[0.1282, 0.5631]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.9422]\", \"lift_point\": \"[0.062, 0.9422]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71177_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4997, 0.6738]\", \"lift_point\": \"[0.4997, 0.6738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.5631]\", \"lift_point\": \"[0.1282, 0.5631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.9422]\", \"lift_point\": \"[0.062, 0.9422]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71177_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4997, 0.6738]\", \"lift_point\": \"[0.4997, 0.6738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.5631]\", \"lift_point\": \"[0.1282, 0.5631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.9422]\", \"lift_point\": \"[0.062, 0.9422]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.3685]\", \"lift_point\": \"[0.1286, 0.3685]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71177_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4997, 0.6738]\", \"lift_point\": \"[0.4997, 0.6738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1282, 0.5631]\", \"lift_point\": \"[0.1282, 0.5631]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.062, 0.9422]\", \"lift_point\": \"[0.062, 0.9422]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1286, 0.3685]\", \"lift_point\": \"[0.1286, 0.3685]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71177_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71445_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3033]\", \"lift_point\": \"[0.8814, 0.3033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71445_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3033]\", \"lift_point\": \"[0.8814, 0.3033]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71445_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3033]\", \"lift_point\": \"[0.8814, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2009, 0.2577]\", \"lift_point\": \"[0.2009, 0.2577]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71445_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3033]\", \"lift_point\": \"[0.8814, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2009, 0.2577]\", \"lift_point\": \"[0.2009, 0.2577]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.2933]\", \"lift_point\": \"[0.325, 0.2933]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71445_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3033]\", \"lift_point\": \"[0.8814, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2009, 0.2577]\", \"lift_point\": \"[0.2009, 0.2577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.2933]\", \"lift_point\": \"[0.325, 0.2933]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3015, 0.8272]\", \"lift_point\": \"[0.3015, 0.8272]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71445_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.3033]\", \"lift_point\": \"[0.8814, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Dallas?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2009, 0.2577]\", \"lift_point\": \"[0.2009, 0.2577]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.2933]\", \"lift_point\": \"[0.325, 0.2933]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3015, 0.8272]\", \"lift_point\": \"[0.3015, 0.8272]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Dallas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71445_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.0996]\", \"lift_point\": \"[0.7912, 0.0996]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71869_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.0996]\", \"lift_point\": \"[0.7912, 0.0996]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.1125]\", \"lift_point\": \"[0.7988, 0.1125]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71869_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.0996]\", \"lift_point\": \"[0.7912, 0.0996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.1125]\", \"lift_point\": \"[0.7988, 0.1125]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5981]\", \"lift_point\": \"[0.0665, 0.5981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71869_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.0996]\", \"lift_point\": \"[0.7912, 0.0996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.1125]\", \"lift_point\": \"[0.7988, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5981]\", \"lift_point\": \"[0.0665, 0.5981]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71869_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.0996]\", \"lift_point\": \"[0.7912, 0.0996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.1125]\", \"lift_point\": \"[0.7988, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5981]\", \"lift_point\": \"[0.0665, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.5226]\", \"lift_point\": \"[0.1365, 0.5226]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71869_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.0996]\", \"lift_point\": \"[0.7912, 0.0996]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.1125]\", \"lift_point\": \"[0.7988, 0.1125]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.5981]\", \"lift_point\": \"[0.0665, 0.5981]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new products by Samsung\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1365, 0.5226]\", \"lift_point\": \"[0.1365, 0.5226]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71869_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7912, 0.1291]\", \"lift_point\": \"[0.7912, 0.1291]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4659]\", \"lift_point\": \"[0.0656, 0.4659]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9065]\", \"lift_point\": \"[0.0649, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" nearest electronics store thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2078, 0.3987]\", \"lift_point\": \"[0.2078, 0.3987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4387, 0.414]\", \"lift_point\": \"[0.4387, 0.414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.7796]\", \"lift_point\": \"[0.1337, 0.7796]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.7796]\", \"lift_point\": \"[0.1337, 0.7796]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.7713]\", \"lift_point\": \"[0.1258, 0.7713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.7796]\", \"lift_point\": \"[0.1337, 0.7796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.7713]\", \"lift_point\": \"[0.1258, 0.7713]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.6526]\", \"lift_point\": \"[0.1787, 0.6526]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.7796]\", \"lift_point\": \"[0.1337, 0.7796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.7713]\", \"lift_point\": \"[0.1258, 0.7713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.6526]\", \"lift_point\": \"[0.1787, 0.6526]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.204, 0.7096]\", \"lift_point\": \"[0.204, 0.7096]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0647, 0.6727]\", \"lift_point\": \"[0.0647, 0.6727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.7796]\", \"lift_point\": \"[0.1337, 0.7796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.7713]\", \"lift_point\": \"[0.1258, 0.7713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.6526]\", \"lift_point\": \"[0.1787, 0.6526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.204, 0.7096]\", \"lift_point\": \"[0.204, 0.7096]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" open now\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3607]\", \"lift_point\": \"[0.1315, 0.3607]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.7796]\", \"lift_point\": \"[0.1337, 0.7796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.7713]\", \"lift_point\": \"[0.1258, 0.7713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.6526]\", \"lift_point\": \"[0.1787, 0.6526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.204, 0.7096]\", \"lift_point\": \"[0.204, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" open now\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1936, 0.6753]\", \"lift_point\": \"[0.1936, 0.6753]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.137, 0.6835]\", \"lift_point\": \"[0.137, 0.6835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" thats open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1337, 0.7796]\", \"lift_point\": \"[0.1337, 0.7796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1258, 0.7713]\", \"lift_point\": \"[0.1258, 0.7713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1787, 0.6526]\", \"lift_point\": \"[0.1787, 0.6526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.204, 0.7096]\", \"lift_point\": \"[0.204, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" open now\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1936, 0.6753]\", \"lift_point\": \"[0.1936, 0.6753]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open now>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71966_28.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3817, 0.683]\", \"lift_point\": \"[0.3817, 0.683]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3817, 0.683]\", \"lift_point\": \"[0.3817, 0.683]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3749, 0.7106]\", \"lift_point\": \"[0.3749, 0.7106]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3817, 0.683]\", \"lift_point\": \"[0.3817, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3749, 0.7106]\", \"lift_point\": \"[0.3749, 0.7106]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0909]\", \"lift_point\": \"[0.0679, 0.0909]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3817, 0.683]\", \"lift_point\": \"[0.3817, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3749, 0.7106]\", \"lift_point\": \"[0.3749, 0.7106]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0909]\", \"lift_point\": \"[0.0679, 0.0909]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.4987]\", \"lift_point\": \"[0.891, 0.4987]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3817, 0.683]\", \"lift_point\": \"[0.3817, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3749, 0.7106]\", \"lift_point\": \"[0.3749, 0.7106]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0909]\", \"lift_point\": \"[0.0679, 0.0909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.4987]\", \"lift_point\": \"[0.891, 0.4987]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6264, 0.5177]\", \"lift_point\": \"[0.6264, 0.5177]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3817, 0.683]\", \"lift_point\": \"[0.3817, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3749, 0.7106]\", \"lift_point\": \"[0.3749, 0.7106]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0909]\", \"lift_point\": \"[0.0679, 0.0909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.4987]\", \"lift_point\": \"[0.891, 0.4987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6264, 0.5177]\", \"lift_point\": \"[0.6264, 0.5177]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6934, 0.7509]\", \"lift_point\": \"[0.6934, 0.7509]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3806, 0.683]\", \"lift_point\": \"[0.3806, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3817, 0.683]\", \"lift_point\": \"[0.3817, 0.683]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3749, 0.7106]\", \"lift_point\": \"[0.3749, 0.7106]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0679, 0.0909]\", \"lift_point\": \"[0.0679, 0.0909]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.891, 0.4987]\", \"lift_point\": \"[0.891, 0.4987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6264, 0.5177]\", \"lift_point\": \"[0.6264, 0.5177]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6934, 0.7509]\", \"lift_point\": \"[0.6934, 0.7509]\", \"typed_text\": \"\" Goal: Set an alarm for 6pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/71997_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72227_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.797, 0.6975]\", \"lift_point\": \"[0.797, 0.6975]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72227_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.797, 0.6975]\", \"lift_point\": \"[0.797, 0.6975]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72227_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.797, 0.6975]\", \"lift_point\": \"[0.797, 0.6975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.4727]\", \"lift_point\": \"[0.0671, 0.4727]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72227_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.797, 0.6975]\", \"lift_point\": \"[0.797, 0.6975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.4727]\", \"lift_point\": \"[0.0671, 0.4727]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72227_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.797, 0.6975]\", \"lift_point\": \"[0.797, 0.6975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.4727]\", \"lift_point\": \"[0.0671, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1223, 0.632]\", \"lift_point\": \"[0.1223, 0.632]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72227_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.797, 0.6975]\", \"lift_point\": \"[0.797, 0.6975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0671, 0.4727]\", \"lift_point\": \"[0.0671, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1223, 0.632]\", \"lift_point\": \"[0.1223, 0.632]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72227_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5182]\", \"lift_point\": \"[0.8924, 0.5182]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72457_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5182]\", \"lift_point\": \"[0.8924, 0.5182]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Katy Perry\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72457_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5182]\", \"lift_point\": \"[0.8924, 0.5182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Katy Perry\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.3846]\", \"lift_point\": \"[0.1234, 0.3846]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72457_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5182]\", \"lift_point\": \"[0.8924, 0.5182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Katy Perry\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.3846]\", \"lift_point\": \"[0.1234, 0.3846]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72457_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5182]\", \"lift_point\": \"[0.8924, 0.5182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Katy Perry\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.3846]\", \"lift_point\": \"[0.1234, 0.3846]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2617, 0.2247]\", \"lift_point\": \"[0.2617, 0.2247]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72457_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.5182]\", \"lift_point\": \"[0.8924, 0.5182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Katy Perry\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1234, 0.3846]\", \"lift_point\": \"[0.1234, 0.3846]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2617, 0.2247]\", \"lift_point\": \"[0.2617, 0.2247]\", \"typed_text\": \"\" Goal: Play the new Katy Perry video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72457_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new mascara>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72530_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8663, 0.51]\", \"lift_point\": \"[0.8663, 0.51]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72530_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8663, 0.51]\", \"lift_point\": \"[0.8663, 0.51]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72530_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8663, 0.51]\", \"lift_point\": \"[0.8663, 0.51]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1284, 0.2329]\", \"lift_point\": \"[0.1284, 0.2329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72530_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8663, 0.51]\", \"lift_point\": \"[0.8663, 0.51]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new mascara\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1284, 0.2329]\", \"lift_point\": \"[0.1284, 0.2329]\", \"typed_text\": \"\" Goal: Search for a new mascara>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72530_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Moscow?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72542_1.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72595_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8791, 0.2315]\", \"lift_point\": \"[0.8791, 0.2315]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72595_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8791, 0.2315]\", \"lift_point\": \"[0.8791, 0.2315]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from NYC to Chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72595_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8791, 0.2315]\", \"lift_point\": \"[0.8791, 0.2315]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from NYC to Chicago\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.3735]\", \"lift_point\": \"[0.1261, 0.3735]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72595_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8791, 0.2315]\", \"lift_point\": \"[0.8791, 0.2315]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from NYC to Chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1261, 0.3735]\", \"lift_point\": \"[0.1261, 0.3735]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Chicago>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72595_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4926]\", \"lift_point\": \"[0.9117, 0.4926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72602_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4926]\", \"lift_point\": \"[0.9117, 0.4926]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9212, 0.461]\", \"lift_point\": \"[0.9212, 0.461]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72602_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4926]\", \"lift_point\": \"[0.9117, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9212, 0.461]\", \"lift_point\": \"[0.9212, 0.461]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72602_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4926]\", \"lift_point\": \"[0.9117, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9212, 0.461]\", \"lift_point\": \"[0.9212, 0.461]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.5746]\", \"lift_point\": \"[0.9, 0.5746]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72602_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4926]\", \"lift_point\": \"[0.9117, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9212, 0.461]\", \"lift_point\": \"[0.9212, 0.461]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.5746]\", \"lift_point\": \"[0.9, 0.5746]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open tomorrow\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72602_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4926]\", \"lift_point\": \"[0.9117, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9212, 0.461]\", \"lift_point\": \"[0.9212, 0.461]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.5746]\", \"lift_point\": \"[0.9, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open tomorrow\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.6431]\", \"lift_point\": \"[0.1231, 0.6431]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72602_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9117, 0.4926]\", \"lift_point\": \"[0.9117, 0.4926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9212, 0.461]\", \"lift_point\": \"[0.9212, 0.461]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9, 0.5746]\", \"lift_point\": \"[0.9, 0.5746]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open tomorrow\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1231, 0.6431]\", \"lift_point\": \"[0.1231, 0.6431]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72602_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5572]\", \"lift_point\": \"[0.8932, 0.5572]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72649_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5572]\", \"lift_point\": \"[0.8932, 0.5572]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in New York\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72649_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5572]\", \"lift_point\": \"[0.8932, 0.5572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in New York\" Goal: How much does a 3 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1214, 0.3778]\", \"lift_point\": \"[0.1214, 0.3778]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72649_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8932, 0.5572]\", \"lift_point\": \"[0.8932, 0.5572]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in New York\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1214, 0.3778]\", \"lift_point\": \"[0.1214, 0.3778]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72649_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72678_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9075, 0.4624]\", \"lift_point\": \"[0.9075, 0.4624]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72678_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9075, 0.4624]\", \"lift_point\": \"[0.9075, 0.4624]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72678_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9075, 0.4624]\", \"lift_point\": \"[0.9075, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1007, 0.4948]\", \"lift_point\": \"[0.1007, 0.4948]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72678_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9075, 0.4624]\", \"lift_point\": \"[0.9075, 0.4624]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the top post on reddit right now?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1007, 0.4948]\", \"lift_point\": \"[0.1007, 0.4948]\", \"typed_text\": \"\" Goal: What's the top post on reddit right now?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72678_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8714, 0.3907]\", \"lift_point\": \"[0.8714, 0.3907]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72686_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8714, 0.3907]\", \"lift_point\": \"[0.8714, 0.3907]\", \"typed_text\": \"\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good BBQ restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72686_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8714, 0.3907]\", \"lift_point\": \"[0.8714, 0.3907]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good BBQ restaurants\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1095, 0.1739]\", \"lift_point\": \"[0.1095, 0.1739]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72686_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8714, 0.3907]\", \"lift_point\": \"[0.8714, 0.3907]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for good BBQ restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1095, 0.1739]\", \"lift_point\": \"[0.1095, 0.1739]\", \"typed_text\": \"\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72686_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8948, 0.6582]\", \"lift_point\": \"[0.8948, 0.6582]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72757_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8948, 0.6582]\", \"lift_point\": \"[0.8948, 0.6582]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72757_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8948, 0.6582]\", \"lift_point\": \"[0.8948, 0.6582]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.3243]\", \"lift_point\": \"[0.1278, 0.3243]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72757_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8948, 0.6582]\", \"lift_point\": \"[0.8948, 0.6582]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1278, 0.3243]\", \"lift_point\": \"[0.1278, 0.3243]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72757_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72842_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1565, 0.4117]\", \"lift_point\": \"[0.1565, 0.4117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72842_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1565, 0.4117]\", \"lift_point\": \"[0.1565, 0.4117]\", \"typed_text\": \"\" Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7707, 0.6928]\", \"lift_point\": \"[0.7707, 0.6928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72842_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1565, 0.4117]\", \"lift_point\": \"[0.1565, 0.4117]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7707, 0.6928]\", \"lift_point\": \"[0.7707, 0.6928]\", \"typed_text\": \"\" Goal: Open a new incognito tab in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/72842_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [PRESS_HOME,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73138_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73138_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.8693]\", \"lift_point\": \"[0.0676, 0.8693]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.8693]\", \"lift_point\": \"[0.0676, 0.8693]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4217]\", \"lift_point\": \"[0.0656, 0.4217]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.8693]\", \"lift_point\": \"[0.0676, 0.8693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4217]\", \"lift_point\": \"[0.0656, 0.4217]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.8693]\", \"lift_point\": \"[0.0676, 0.8693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4217]\", \"lift_point\": \"[0.0656, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway? \" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2779, 0.4618]\", \"lift_point\": \"[0.2779, 0.4618]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.8693]\", \"lift_point\": \"[0.0676, 0.8693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4217]\", \"lift_point\": \"[0.0656, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2779, 0.4618]\", \"lift_point\": \"[0.2779, 0.4618]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.8693]\", \"lift_point\": \"[0.0676, 0.8693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4217]\", \"lift_point\": \"[0.0656, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2779, 0.4618]\", \"lift_point\": \"[0.2779, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7503, 0.433]\", \"lift_point\": \"[0.7503, 0.433]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5049, 0.6656]\", \"lift_point\": \"[0.5049, 0.6656]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.303, 0.7519]\", \"lift_point\": \"[0.303, 0.7519]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.8693]\", \"lift_point\": \"[0.0676, 0.8693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0656, 0.4217]\", \"lift_point\": \"[0.0656, 0.4217]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2779, 0.4618]\", \"lift_point\": \"[0.2779, 0.4618]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7503, 0.433]\", \"lift_point\": \"[0.7503, 0.433]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73328_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.8518]\", \"lift_point\": \"[0.5058, 0.8518]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.8518]\", \"lift_point\": \"[0.5058, 0.8518]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1346, 0.6041]\", \"lift_point\": \"[0.1346, 0.6041]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.8518]\", \"lift_point\": \"[0.5058, 0.8518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1346, 0.6041]\", \"lift_point\": \"[0.1346, 0.6041]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.9086]\", \"lift_point\": \"[0.0618, 0.9086]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.8518]\", \"lift_point\": \"[0.5058, 0.8518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1346, 0.6041]\", \"lift_point\": \"[0.1346, 0.6041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.9086]\", \"lift_point\": \"[0.0618, 0.9086]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.5857]\", \"lift_point\": \"[0.0615, 0.5857]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.8518]\", \"lift_point\": \"[0.5058, 0.8518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1346, 0.6041]\", \"lift_point\": \"[0.1346, 0.6041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.9086]\", \"lift_point\": \"[0.0618, 0.9086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.5857]\", \"lift_point\": \"[0.0615, 0.5857]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Lowes? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.8518]\", \"lift_point\": \"[0.5058, 0.8518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1346, 0.6041]\", \"lift_point\": \"[0.1346, 0.6041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.9086]\", \"lift_point\": \"[0.0618, 0.9086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.5857]\", \"lift_point\": \"[0.0615, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Lowes? \" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1561, 0.4291]\", \"lift_point\": \"[0.1561, 0.4291]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5058, 0.8518]\", \"lift_point\": \"[0.5058, 0.8518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1346, 0.6041]\", \"lift_point\": \"[0.1346, 0.6041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.9086]\", \"lift_point\": \"[0.0618, 0.9086]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0615, 0.5857]\", \"lift_point\": \"[0.0615, 0.5857]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Lowes? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1561, 0.4291]\", \"lift_point\": \"[0.1561, 0.4291]\", \"typed_text\": \"\" Goal: How do I get to the nearest Lowe's?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73511_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6494, 0.8585]\", \"lift_point\": \"[0.6494, 0.8585]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73949_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6494, 0.8585]\", \"lift_point\": \"[0.6494, 0.8585]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.8097]\", \"lift_point\": \"[0.0538, 0.8097]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73949_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6494, 0.8585]\", \"lift_point\": \"[0.6494, 0.8585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.8097]\", \"lift_point\": \"[0.0538, 0.8097]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new justin bieber video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73949_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6494, 0.8585]\", \"lift_point\": \"[0.6494, 0.8585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.8097]\", \"lift_point\": \"[0.0538, 0.8097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new justin bieber video\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1114, 0.425]\", \"lift_point\": \"[0.1114, 0.425]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73949_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6494, 0.8585]\", \"lift_point\": \"[0.6494, 0.8585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.8097]\", \"lift_point\": \"[0.0538, 0.8097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new justin bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1114, 0.425]\", \"lift_point\": \"[0.1114, 0.425]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.507, 0.5601]\", \"lift_point\": \"[0.507, 0.5601]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73949_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6494, 0.8585]\", \"lift_point\": \"[0.6494, 0.8585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.8097]\", \"lift_point\": \"[0.0538, 0.8097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new justin bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1114, 0.425]\", \"lift_point\": \"[0.1114, 0.425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.507, 0.5601]\", \"lift_point\": \"[0.507, 0.5601]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2132, 0.8475]\", \"lift_point\": \"[0.2132, 0.8475]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73949_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6494, 0.8585]\", \"lift_point\": \"[0.6494, 0.8585]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0538, 0.8097]\", \"lift_point\": \"[0.0538, 0.8097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new justin bieber video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1114, 0.425]\", \"lift_point\": \"[0.1114, 0.425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.507, 0.5601]\", \"lift_point\": \"[0.507, 0.5601]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2132, 0.8475]\", \"lift_point\": \"[0.2132, 0.8475]\", \"typed_text\": \"\" Goal: Play the new Justin Bieber video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73949_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73956_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73956_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6587, 0.3629]\", \"lift_point\": \"[0.6587, 0.3629]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73956_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6587, 0.3629]\", \"lift_point\": \"[0.6587, 0.3629]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.5655]\", \"lift_point\": \"[0.0553, 0.5655]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73956_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6587, 0.3629]\", \"lift_point\": \"[0.6587, 0.3629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.5655]\", \"lift_point\": \"[0.0553, 0.5655]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in washington DC\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73956_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6587, 0.3629]\", \"lift_point\": \"[0.6587, 0.3629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.5655]\", \"lift_point\": \"[0.0553, 0.5655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in washington DC\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.5141]\", \"lift_point\": \"[0.1201, 0.5141]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73956_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6587, 0.3629]\", \"lift_point\": \"[0.6587, 0.3629]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0553, 0.5655]\", \"lift_point\": \"[0.0553, 0.5655]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1201, 0.5141]\", \"lift_point\": \"[0.1201, 0.5141]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73956_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6405]\", \"lift_point\": \"[0.7794, 0.6405]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73995_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6405]\", \"lift_point\": \"[0.7794, 0.6405]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.039, 0.2988]\", \"lift_point\": \"[0.039, 0.2988]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73995_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6405]\", \"lift_point\": \"[0.7794, 0.6405]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.039, 0.2988]\", \"lift_point\": \"[0.039, 0.2988]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73995_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6405]\", \"lift_point\": \"[0.7794, 0.6405]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.039, 0.2988]\", \"lift_point\": \"[0.039, 0.2988]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.481]\", \"lift_point\": \"[0.1161, 0.481]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73995_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6405]\", \"lift_point\": \"[0.7794, 0.6405]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.039, 0.2988]\", \"lift_point\": \"[0.039, 0.2988]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1161, 0.481]\", \"lift_point\": \"[0.1161, 0.481]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/73995_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.8316]\", \"lift_point\": \"[0.0473, 0.8316]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.8316]\", \"lift_point\": \"[0.0473, 0.8316]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the latest video from the Washington Post\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.8316]\", \"lift_point\": \"[0.0473, 0.8316]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the latest video from the Washington Post\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.6087]\", \"lift_point\": \"[0.1008, 0.6087]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.8316]\", \"lift_point\": \"[0.0473, 0.8316]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the latest video from the Washington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.6087]\", \"lift_point\": \"[0.1008, 0.6087]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.6696]\", \"lift_point\": \"[0.4026, 0.6696]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.8316]\", \"lift_point\": \"[0.0473, 0.8316]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the latest video from the Washington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.6087]\", \"lift_point\": \"[0.1008, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.6696]\", \"lift_point\": \"[0.4026, 0.6696]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2946, 0.92]\", \"lift_point\": \"[0.2946, 0.92]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9114, 0.3703]\", \"lift_point\": \"[0.9114, 0.3703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.8316]\", \"lift_point\": \"[0.0473, 0.8316]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the latest video from the Washington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.6087]\", \"lift_point\": \"[0.1008, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.6696]\", \"lift_point\": \"[0.4026, 0.6696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2946, 0.92]\", \"lift_point\": \"[0.2946, 0.92]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2129, 0.8447]\", \"lift_point\": \"[0.2129, 0.8447]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.0742]\", \"lift_point\": \"[0.0578, 0.0742]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8712, 0.9233]\", \"lift_point\": \"[0.8712, 0.9233]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0473, 0.8316]\", \"lift_point\": \"[0.0473, 0.8316]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"the latest video from the Washington Post\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1008, 0.6087]\", \"lift_point\": \"[0.1008, 0.6087]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.6696]\", \"lift_point\": \"[0.4026, 0.6696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2946, 0.92]\", \"lift_point\": \"[0.2946, 0.92]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2129, 0.8447]\", \"lift_point\": \"[0.2129, 0.8447]\", \"typed_text\": \"\" Goal: Play the latest video from the Washington Post>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74037_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74118_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3741]\", \"lift_point\": \"[0.0582, 0.3741]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74118_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3741]\", \"lift_point\": \"[0.0582, 0.3741]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Denver?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74118_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3741]\", \"lift_point\": \"[0.0582, 0.3741]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Denver?\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0573, 0.3743]\", \"lift_point\": \"[0.0573, 0.3743]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74118_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3741]\", \"lift_point\": \"[0.0582, 0.3741]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Denver?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0573, 0.3743]\", \"lift_point\": \"[0.0573, 0.3743]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.3796]\", \"lift_point\": \"[0.1255, 0.3796]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74118_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3741]\", \"lift_point\": \"[0.0582, 0.3741]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Denver?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0573, 0.3743]\", \"lift_point\": \"[0.0573, 0.3743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.3796]\", \"lift_point\": \"[0.1255, 0.3796]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3032, 0.501]\", \"lift_point\": \"[0.3032, 0.501]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74118_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.5928]\", \"lift_point\": \"[0.7798, 0.5928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3741]\", \"lift_point\": \"[0.0582, 0.3741]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 2 bedroom apartment rent for in Denver?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0573, 0.3743]\", \"lift_point\": \"[0.0573, 0.3743]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.3796]\", \"lift_point\": \"[0.1255, 0.3796]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3032, 0.501]\", \"lift_point\": \"[0.3032, 0.501]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74118_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6608, 0.3757]\", \"lift_point\": \"[0.6608, 0.3757]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6608, 0.3757]\", \"lift_point\": \"[0.6608, 0.3757]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.2999]\", \"lift_point\": \"[0.0655, 0.2999]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6608, 0.3757]\", \"lift_point\": \"[0.6608, 0.3757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.2999]\", \"lift_point\": \"[0.0655, 0.2999]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0739, 0.3835]\", \"lift_point\": \"[0.0739, 0.3835]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6608, 0.3757]\", \"lift_point\": \"[0.6608, 0.3757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.2999]\", \"lift_point\": \"[0.0655, 0.2999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0739, 0.3835]\", \"lift_point\": \"[0.0739, 0.3835]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"5 star sushi restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6608, 0.3757]\", \"lift_point\": \"[0.6608, 0.3757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.2999]\", \"lift_point\": \"[0.0655, 0.2999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0739, 0.3835]\", \"lift_point\": \"[0.0739, 0.3835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"5 star sushi restaurants\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.4443]\", \"lift_point\": \"[0.1255, 0.4443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6608, 0.3757]\", \"lift_point\": \"[0.6608, 0.3757]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0655, 0.2999]\", \"lift_point\": \"[0.0655, 0.2999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0739, 0.3835]\", \"lift_point\": \"[0.0739, 0.3835]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"5 star sushi restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1255, 0.4443]\", \"lift_point\": \"[0.1255, 0.4443]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74295_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74325_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9094, 0.3443]\", \"lift_point\": \"[0.9094, 0.3443]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74325_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9094, 0.3443]\", \"lift_point\": \"[0.9094, 0.3443]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0784, 0.3589]\", \"lift_point\": \"[0.0784, 0.3589]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74325_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9094, 0.3443]\", \"lift_point\": \"[0.9094, 0.3443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0784, 0.3589]\", \"lift_point\": \"[0.0784, 0.3589]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Korean restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74325_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9094, 0.3443]\", \"lift_point\": \"[0.9094, 0.3443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0784, 0.3589]\", \"lift_point\": \"[0.0784, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Korean restaurants\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8038, 0.7325]\", \"lift_point\": \"[0.8038, 0.7325]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74325_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9094, 0.3443]\", \"lift_point\": \"[0.9094, 0.3443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0784, 0.3589]\", \"lift_point\": \"[0.0784, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Korean restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8038, 0.7325]\", \"lift_point\": \"[0.8038, 0.7325]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1334, 0.59]\", \"lift_point\": \"[0.1334, 0.59]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74325_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9094, 0.3443]\", \"lift_point\": \"[0.9094, 0.3443]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0784, 0.3589]\", \"lift_point\": \"[0.0784, 0.3589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Korean restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8038, 0.7325]\", \"lift_point\": \"[0.8038, 0.7325]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1334, 0.59]\", \"lift_point\": \"[0.1334, 0.59]\", \"typed_text\": \"\" Goal: Search for good Korean restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74325_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9418, 0.3691]\", \"lift_point\": \"[0.9418, 0.3691]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9418, 0.3691]\", \"lift_point\": \"[0.9418, 0.3691]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9481, 0.452]\", \"lift_point\": \"[0.9481, 0.452]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9418, 0.3691]\", \"lift_point\": \"[0.9418, 0.3691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9481, 0.452]\", \"lift_point\": \"[0.9481, 0.452]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.2436]\", \"lift_point\": \"[0.0578, 0.2436]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9418, 0.3691]\", \"lift_point\": \"[0.9418, 0.3691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9481, 0.452]\", \"lift_point\": \"[0.9481, 0.452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.2436]\", \"lift_point\": \"[0.0578, 0.2436]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of the United States\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9418, 0.3691]\", \"lift_point\": \"[0.9418, 0.3691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9481, 0.452]\", \"lift_point\": \"[0.9481, 0.452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.2436]\", \"lift_point\": \"[0.0578, 0.2436]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of the United States\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7726, 0.7532]\", \"lift_point\": \"[0.7726, 0.7532]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9418, 0.3691]\", \"lift_point\": \"[0.9418, 0.3691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9481, 0.452]\", \"lift_point\": \"[0.9481, 0.452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.2436]\", \"lift_point\": \"[0.0578, 0.2436]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of the United States\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7726, 0.7532]\", \"lift_point\": \"[0.7726, 0.7532]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1146, 0.4143]\", \"lift_point\": \"[0.1146, 0.4143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9165, 0.3032]\", \"lift_point\": \"[0.9165, 0.3032]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9418, 0.3691]\", \"lift_point\": \"[0.9418, 0.3691]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9481, 0.452]\", \"lift_point\": \"[0.9481, 0.452]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0578, 0.2436]\", \"lift_point\": \"[0.0578, 0.2436]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Google the capital of the United States\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7726, 0.7532]\", \"lift_point\": \"[0.7726, 0.7532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1146, 0.4143]\", \"lift_point\": \"[0.1146, 0.4143]\", \"typed_text\": \"\" Goal: Google the capital of the United States>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74352_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my email inbox>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.178, 0.3729]\", \"lift_point\": \"[0.178, 0.3729]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.178, 0.3729]\", \"lift_point\": \"[0.178, 0.3729]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.956, 0.4835]\", \"lift_point\": \"[0.956, 0.4835]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.178, 0.3729]\", \"lift_point\": \"[0.178, 0.3729]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.956, 0.4835]\", \"lift_point\": \"[0.956, 0.4835]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9521, 0.559]\", \"lift_point\": \"[0.9521, 0.559]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.178, 0.3729]\", \"lift_point\": \"[0.178, 0.3729]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.956, 0.4835]\", \"lift_point\": \"[0.956, 0.4835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9521, 0.559]\", \"lift_point\": \"[0.9521, 0.559]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.895]\", \"lift_point\": \"[0.4178, 0.895]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.178, 0.3729]\", \"lift_point\": \"[0.178, 0.3729]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.956, 0.4835]\", \"lift_point\": \"[0.956, 0.4835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9521, 0.559]\", \"lift_point\": \"[0.9521, 0.559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.895]\", \"lift_point\": \"[0.4178, 0.895]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5664, 0.7445]\", \"lift_point\": \"[0.5664, 0.7445]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.178, 0.3729]\", \"lift_point\": \"[0.178, 0.3729]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.956, 0.4835]\", \"lift_point\": \"[0.956, 0.4835]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9521, 0.559]\", \"lift_point\": \"[0.9521, 0.559]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4178, 0.895]\", \"lift_point\": \"[0.4178, 0.895]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5664, 0.7445]\", \"lift_point\": \"[0.5664, 0.7445]\", \"typed_text\": \"\" Goal: Check my email inbox>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74389_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Twitter app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.3677]\", \"lift_point\": \"[0.804, 0.3677]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.3677]\", \"lift_point\": \"[0.804, 0.3677]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.8029]\", \"lift_point\": \"[0.0591, 0.8029]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.3677]\", \"lift_point\": \"[0.804, 0.3677]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.8029]\", \"lift_point\": \"[0.0591, 0.8029]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0896, 0.3765]\", \"lift_point\": \"[0.0896, 0.3765]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.3677]\", \"lift_point\": \"[0.804, 0.3677]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.8029]\", \"lift_point\": \"[0.0591, 0.8029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0896, 0.3765]\", \"lift_point\": \"[0.0896, 0.3765]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.3677]\", \"lift_point\": \"[0.804, 0.3677]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.8029]\", \"lift_point\": \"[0.0591, 0.8029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0896, 0.3765]\", \"lift_point\": \"[0.0896, 0.3765]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.2468]\", \"lift_point\": \"[0.121, 0.2468]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.3677]\", \"lift_point\": \"[0.804, 0.3677]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.8029]\", \"lift_point\": \"[0.0591, 0.8029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0896, 0.3765]\", \"lift_point\": \"[0.0896, 0.3765]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.2468]\", \"lift_point\": \"[0.121, 0.2468]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.7832]\", \"lift_point\": \"[0.1461, 0.7832]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.804, 0.3677]\", \"lift_point\": \"[0.804, 0.3677]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.8029]\", \"lift_point\": \"[0.0591, 0.8029]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0896, 0.3765]\", \"lift_point\": \"[0.0896, 0.3765]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"twitter\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.121, 0.2468]\", \"lift_point\": \"[0.121, 0.2468]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1461, 0.7832]\", \"lift_point\": \"[0.1461, 0.7832]\", \"typed_text\": \"\" Goal: Install the Twitter app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74441_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in San Francisco>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9596, 0.3367]\", \"lift_point\": \"[0.9596, 0.3367]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74447_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9596, 0.3367]\", \"lift_point\": \"[0.9596, 0.3367]\", \"typed_text\": \"\" Goal: Search for hotels in San Francisco>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9539, 0.3271]\", \"lift_point\": \"[0.9539, 0.3271]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74447_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9596, 0.3367]\", \"lift_point\": \"[0.9596, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9539, 0.3271]\", \"lift_point\": \"[0.9539, 0.3271]\", \"typed_text\": \"\" Goal: Search for hotels in San Francisco>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.2699]\", \"lift_point\": \"[0.0606, 0.2699]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74447_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9596, 0.3367]\", \"lift_point\": \"[0.9596, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9539, 0.3271]\", \"lift_point\": \"[0.9539, 0.3271]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.2699]\", \"lift_point\": \"[0.0606, 0.2699]\", \"typed_text\": \"\" Goal: Search for hotels in San Francisco>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Francisco\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74447_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9596, 0.3367]\", \"lift_point\": \"[0.9596, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9539, 0.3271]\", \"lift_point\": \"[0.9539, 0.3271]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.2699]\", \"lift_point\": \"[0.0606, 0.2699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Francisco\" Goal: Search for hotels in San Francisco>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.11, 0.401]\", \"lift_point\": \"[0.11, 0.401]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74447_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9596, 0.3367]\", \"lift_point\": \"[0.9596, 0.3367]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9539, 0.3271]\", \"lift_point\": \"[0.9539, 0.3271]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.2699]\", \"lift_point\": \"[0.0606, 0.2699]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Francisco\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.11, 0.401]\", \"lift_point\": \"[0.11, 0.401]\", \"typed_text\": \"\" Goal: Search for hotels in San Francisco>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74447_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74501_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9191, 0.4716]\", \"lift_point\": \"[0.9191, 0.4716]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74501_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9191, 0.4716]\", \"lift_point\": \"[0.9191, 0.4716]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.3762]\", \"lift_point\": \"[0.0544, 0.3762]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74501_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9191, 0.4716]\", \"lift_point\": \"[0.9191, 0.4716]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.3762]\", \"lift_point\": \"[0.0544, 0.3762]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Phoenix?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74501_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9191, 0.4716]\", \"lift_point\": \"[0.9191, 0.4716]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.3762]\", \"lift_point\": \"[0.0544, 0.3762]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Phoenix?\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.4282]\", \"lift_point\": \"[0.1142, 0.4282]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74501_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9191, 0.4716]\", \"lift_point\": \"[0.9191, 0.4716]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.3762]\", \"lift_point\": \"[0.0544, 0.3762]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Phoenix?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.4282]\", \"lift_point\": \"[0.1142, 0.4282]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74501_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9573, 0.3885]\", \"lift_point\": \"[0.9573, 0.3885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74540_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9573, 0.3885]\", \"lift_point\": \"[0.9573, 0.3885]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4132]\", \"lift_point\": \"[0.0576, 0.4132]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74540_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9573, 0.3885]\", \"lift_point\": \"[0.9573, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4132]\", \"lift_point\": \"[0.0576, 0.4132]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74540_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9573, 0.3885]\", \"lift_point\": \"[0.9573, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4132]\", \"lift_point\": \"[0.0576, 0.4132]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.595]\", \"lift_point\": \"[0.1291, 0.595]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74540_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9573, 0.3885]\", \"lift_point\": \"[0.9573, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4132]\", \"lift_point\": \"[0.0576, 0.4132]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.595]\", \"lift_point\": \"[0.1291, 0.595]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.3789]\", \"lift_point\": \"[0.3197, 0.3789]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74540_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9573, 0.3885]\", \"lift_point\": \"[0.9573, 0.3885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0576, 0.4132]\", \"lift_point\": \"[0.0576, 0.4132]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1291, 0.595]\", \"lift_point\": \"[0.1291, 0.595]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3197, 0.3789]\", \"lift_point\": \"[0.3197, 0.3789]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74540_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9531, 0.4794]\", \"lift_point\": \"[0.9531, 0.4794]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74555_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9531, 0.4794]\", \"lift_point\": \"[0.9531, 0.4794]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.273]\", \"lift_point\": \"[0.0665, 0.273]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74555_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9531, 0.4794]\", \"lift_point\": \"[0.9531, 0.4794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.273]\", \"lift_point\": \"[0.0665, 0.273]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74555_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9531, 0.4794]\", \"lift_point\": \"[0.9531, 0.4794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.273]\", \"lift_point\": \"[0.0665, 0.273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6992, 0.5088]\", \"lift_point\": \"[0.6992, 0.5088]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74555_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9531, 0.4794]\", \"lift_point\": \"[0.9531, 0.4794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.273]\", \"lift_point\": \"[0.0665, 0.273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6992, 0.5088]\", \"lift_point\": \"[0.6992, 0.5088]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.5932]\", \"lift_point\": \"[0.1294, 0.5932]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74555_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9531, 0.4794]\", \"lift_point\": \"[0.9531, 0.4794]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.273]\", \"lift_point\": \"[0.0665, 0.273]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Home Depot?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6992, 0.5088]\", \"lift_point\": \"[0.6992, 0.5088]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.5932]\", \"lift_point\": \"[0.1294, 0.5932]\", \"typed_text\": \"\" Goal: How do I get to the nearest Home Depot?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74555_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a3 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74564_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a3 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.932, 0.3745]\", \"lift_point\": \"[0.932, 0.3745]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74564_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.932, 0.3745]\", \"lift_point\": \"[0.932, 0.3745]\", \"typed_text\": \"\" Goal: How much does a3 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9368, 0.3616]\", \"lift_point\": \"[0.9368, 0.3616]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74564_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.932, 0.3745]\", \"lift_point\": \"[0.932, 0.3745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9368, 0.3616]\", \"lift_point\": \"[0.9368, 0.3616]\", \"typed_text\": \"\" Goal: How much does a3 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9319, 0.3459]\", \"lift_point\": \"[0.9319, 0.3459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74564_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.932, 0.3745]\", \"lift_point\": \"[0.932, 0.3745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9368, 0.3616]\", \"lift_point\": \"[0.9368, 0.3616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9319, 0.3459]\", \"lift_point\": \"[0.9319, 0.3459]\", \"typed_text\": \"\" Goal: How much does a3 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.2721]\", \"lift_point\": \"[0.0596, 0.2721]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74564_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.932, 0.3745]\", \"lift_point\": \"[0.932, 0.3745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9368, 0.3616]\", \"lift_point\": \"[0.9368, 0.3616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9319, 0.3459]\", \"lift_point\": \"[0.9319, 0.3459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.2721]\", \"lift_point\": \"[0.0596, 0.2721]\", \"typed_text\": \"\" Goal: How much does a3 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1466, 0.6255]\", \"lift_point\": \"[0.1466, 0.6255]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74564_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.932, 0.3745]\", \"lift_point\": \"[0.932, 0.3745]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9368, 0.3616]\", \"lift_point\": \"[0.9368, 0.3616]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9319, 0.3459]\", \"lift_point\": \"[0.9319, 0.3459]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.2721]\", \"lift_point\": \"[0.0596, 0.2721]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1466, 0.6255]\", \"lift_point\": \"[0.1466, 0.6255]\", \"typed_text\": \"\" Goal: How much does a3 bedroom apartment rent for in Portland?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74564_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74581_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9469, 0.433]\", \"lift_point\": \"[0.9469, 0.433]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74581_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9469, 0.433]\", \"lift_point\": \"[0.9469, 0.433]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0746, 0.2911]\", \"lift_point\": \"[0.0746, 0.2911]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74581_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9469, 0.433]\", \"lift_point\": \"[0.9469, 0.433]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0746, 0.2911]\", \"lift_point\": \"[0.0746, 0.2911]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74581_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9469, 0.433]\", \"lift_point\": \"[0.9469, 0.433]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0746, 0.2911]\", \"lift_point\": \"[0.0746, 0.2911]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.5986]\", \"lift_point\": \"[0.1308, 0.5986]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74581_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9469, 0.433]\", \"lift_point\": \"[0.9469, 0.433]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0746, 0.2911]\", \"lift_point\": \"[0.0746, 0.2911]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in San Jose?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1308, 0.5986]\", \"lift_point\": \"[0.1308, 0.5986]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Jose?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74581_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.668]\", \"lift_point\": \"[0.7785, 0.668]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.668]\", \"lift_point\": \"[0.7785, 0.668]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6236]\", \"lift_point\": \"[0.0584, 0.6236]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.668]\", \"lift_point\": \"[0.7785, 0.668]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6236]\", \"lift_point\": \"[0.0584, 0.6236]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9304]\", \"lift_point\": \"[0.0589, 0.9304]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.668]\", \"lift_point\": \"[0.7785, 0.668]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6236]\", \"lift_point\": \"[0.0584, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9304]\", \"lift_point\": \"[0.0589, 0.9304]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.4573]\", \"lift_point\": \"[0.0563, 0.4573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.668]\", \"lift_point\": \"[0.7785, 0.668]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6236]\", \"lift_point\": \"[0.0584, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9304]\", \"lift_point\": \"[0.0589, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.4573]\", \"lift_point\": \"[0.0563, 0.4573]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how much does a 3 bedroom apartment rent for in miami?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.668]\", \"lift_point\": \"[0.7785, 0.668]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6236]\", \"lift_point\": \"[0.0584, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9304]\", \"lift_point\": \"[0.0589, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.4573]\", \"lift_point\": \"[0.0563, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how much does a 3 bedroom apartment rent for in miami?\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1254, 0.6256]\", \"lift_point\": \"[0.1254, 0.6256]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7785, 0.668]\", \"lift_point\": \"[0.7785, 0.668]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6236]\", \"lift_point\": \"[0.0584, 0.6236]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0589, 0.9304]\", \"lift_point\": \"[0.0589, 0.9304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0563, 0.4573]\", \"lift_point\": \"[0.0563, 0.4573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how much does a 3 bedroom apartment rent for in miami?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1254, 0.6256]\", \"lift_point\": \"[0.1254, 0.6256]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74638_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated sushi restaurant>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74787_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for top rated sushi restaurant>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3644, 0.4923]\", \"lift_point\": \"[0.3644, 0.4923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74787_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3644, 0.4923]\", \"lift_point\": \"[0.3644, 0.4923]\", \"typed_text\": \"\" Goal: Search for top rated sushi restaurant>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0417, 0.4664]\", \"lift_point\": \"[0.0417, 0.4664]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74787_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3644, 0.4923]\", \"lift_point\": \"[0.3644, 0.4923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0417, 0.4664]\", \"lift_point\": \"[0.0417, 0.4664]\", \"typed_text\": \"\" Goal: Search for top rated sushi restaurant>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated sushi restaurant\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74787_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3644, 0.4923]\", \"lift_point\": \"[0.3644, 0.4923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0417, 0.4664]\", \"lift_point\": \"[0.0417, 0.4664]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated sushi restaurant\" Goal: Search for top rated sushi restaurant>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74787_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3644, 0.4923]\", \"lift_point\": \"[0.3644, 0.4923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0417, 0.4664]\", \"lift_point\": \"[0.0417, 0.4664]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated sushi restaurant\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for top rated sushi restaurant>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74787_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.4877]\", \"lift_point\": \"[0.3276, 0.4877]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.4877]\", \"lift_point\": \"[0.3276, 0.4877]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.4957]\", \"lift_point\": \"[0.3288, 0.4957]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.4877]\", \"lift_point\": \"[0.3276, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.4957]\", \"lift_point\": \"[0.3288, 0.4957]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5062]\", \"lift_point\": \"[0.3312, 0.5062]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.4877]\", \"lift_point\": \"[0.3276, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.4957]\", \"lift_point\": \"[0.3288, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5062]\", \"lift_point\": \"[0.3312, 0.5062]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.5036]\", \"lift_point\": \"[0.0544, 0.5036]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.4877]\", \"lift_point\": \"[0.3276, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.4957]\", \"lift_point\": \"[0.3288, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5062]\", \"lift_point\": \"[0.3312, 0.5062]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.5036]\", \"lift_point\": \"[0.0544, 0.5036]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.4877]\", \"lift_point\": \"[0.3276, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.4957]\", \"lift_point\": \"[0.3288, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5062]\", \"lift_point\": \"[0.3312, 0.5062]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.5036]\", \"lift_point\": \"[0.0544, 0.5036]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3472, 0.4705]\", \"lift_point\": \"[0.3472, 0.4705]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3276, 0.4877]\", \"lift_point\": \"[0.3276, 0.4877]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3288, 0.4957]\", \"lift_point\": \"[0.3288, 0.4957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3312, 0.5062]\", \"lift_point\": \"[0.3312, 0.5062]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0544, 0.5036]\", \"lift_point\": \"[0.0544, 0.5036]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Paris\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Paris>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74851_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74865_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74865_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.4986]\", \"lift_point\": \"[0.331, 0.4986]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74865_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.4986]\", \"lift_point\": \"[0.331, 0.4986]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.4415]\", \"lift_point\": \"[0.0643, 0.4415]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74865_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.4986]\", \"lift_point\": \"[0.331, 0.4986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.4415]\", \"lift_point\": \"[0.0643, 0.4415]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Diego\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74865_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.4986]\", \"lift_point\": \"[0.331, 0.4986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.4415]\", \"lift_point\": \"[0.0643, 0.4415]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Diego\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74865_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.331, 0.4986]\", \"lift_point\": \"[0.331, 0.4986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0643, 0.4415]\", \"lift_point\": \"[0.0643, 0.4415]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in San Diego\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in San Diego>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74865_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74912_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3427, 0.4952]\", \"lift_point\": \"[0.3427, 0.4952]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74912_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3427, 0.4952]\", \"lift_point\": \"[0.3427, 0.4952]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.3888]\", \"lift_point\": \"[0.0708, 0.3888]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74912_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3427, 0.4952]\", \"lift_point\": \"[0.3427, 0.4952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.3888]\", \"lift_point\": \"[0.0708, 0.3888]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74912_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3427, 0.4952]\", \"lift_point\": \"[0.3427, 0.4952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.3888]\", \"lift_point\": \"[0.0708, 0.3888]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74912_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3427, 0.4952]\", \"lift_point\": \"[0.3427, 0.4952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0708, 0.3888]\", \"lift_point\": \"[0.0708, 0.3888]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does the HisenseTV cost?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the HisenseTV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74912_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new skincare product>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3282, 0.5012]\", \"lift_point\": \"[0.3282, 0.5012]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3282, 0.5012]\", \"lift_point\": \"[0.3282, 0.5012]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0749, 0.3695]\", \"lift_point\": \"[0.0749, 0.3695]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3282, 0.5012]\", \"lift_point\": \"[0.3282, 0.5012]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0749, 0.3695]\", \"lift_point\": \"[0.0749, 0.3695]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new skincare product\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3282, 0.5012]\", \"lift_point\": \"[0.3282, 0.5012]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0749, 0.3695]\", \"lift_point\": \"[0.0749, 0.3695]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new skincare product\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3282, 0.5012]\", \"lift_point\": \"[0.3282, 0.5012]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0749, 0.3695]\", \"lift_point\": \"[0.0749, 0.3695]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new skincare product\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new skincare product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/74920_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75077_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6657, 0.3626]\", \"lift_point\": \"[0.6657, 0.3626]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75077_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6657, 0.3626]\", \"lift_point\": \"[0.6657, 0.3626]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.7812]\", \"lift_point\": \"[0.0619, 0.7812]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75077_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6657, 0.3626]\", \"lift_point\": \"[0.6657, 0.3626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.7812]\", \"lift_point\": \"[0.0619, 0.7812]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.3296]\", \"lift_point\": \"[0.0695, 0.3296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75077_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6657, 0.3626]\", \"lift_point\": \"[0.6657, 0.3626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.7812]\", \"lift_point\": \"[0.0619, 0.7812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.3296]\", \"lift_point\": \"[0.0695, 0.3296]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"5 star sushi restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75077_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6657, 0.3626]\", \"lift_point\": \"[0.6657, 0.3626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.7812]\", \"lift_point\": \"[0.0619, 0.7812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.3296]\", \"lift_point\": \"[0.0695, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"5 star sushi restaurants\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75077_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6657, 0.3626]\", \"lift_point\": \"[0.6657, 0.3626]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0619, 0.7812]\", \"lift_point\": \"[0.0619, 0.7812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0695, 0.3296]\", \"lift_point\": \"[0.0695, 0.3296]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"5 star sushi restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for 5 star sushi restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75077_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9374, 0.3882]\", \"lift_point\": \"[0.9374, 0.3882]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Ariana Grande\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6396, 0.4336]\", \"lift_point\": \"[0.6396, 0.4336]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Ariana Grande\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Ariana Grande\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6132, 0.2649]\", \"lift_point\": \"[0.6132, 0.2649]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0712, 0.6123]\", \"lift_point\": \"[0.0712, 0.6123]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Ariana Grande\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6132, 0.2649]\", \"lift_point\": \"[0.6132, 0.2649]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0945, 0.366]\", \"lift_point\": \"[0.0945, 0.366]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0396, 0.6168]\", \"lift_point\": \"[0.0396, 0.6168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Ariana Grande\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6132, 0.2649]\", \"lift_point\": \"[0.6132, 0.2649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0945, 0.366]\", \"lift_point\": \"[0.0945, 0.366]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2091, 0.3261]\", \"lift_point\": \"[0.2091, 0.3261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Ariana Grande\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6132, 0.2649]\", \"lift_point\": \"[0.6132, 0.2649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0945, 0.366]\", \"lift_point\": \"[0.0945, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2091, 0.3261]\", \"lift_point\": \"[0.2091, 0.3261]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2094, 0.8261]\", \"lift_point\": \"[0.2094, 0.8261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.8267]\", \"lift_point\": \"[0.0662, 0.8267]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Ariana Grande\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6132, 0.2649]\", \"lift_point\": \"[0.6132, 0.2649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0945, 0.366]\", \"lift_point\": \"[0.0945, 0.366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2091, 0.3261]\", \"lift_point\": \"[0.2091, 0.3261]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2094, 0.8261]\", \"lift_point\": \"[0.2094, 0.8261]\", \"typed_text\": \"\" Goal: Play the new Ariana Grande video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75127_17.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new nail polish>55555", + "response": "Action Plan: [PRESS_HOME,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6103]\", \"lift_point\": \"[0.0584, 0.6103]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6103]\", \"lift_point\": \"[0.0584, 0.6103]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a new nail polish\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6103]\", \"lift_point\": \"[0.0584, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a new nail polish\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6103]\", \"lift_point\": \"[0.0584, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a new nail polish\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6103]\", \"lift_point\": \"[0.0584, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a new nail polish\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6103]\", \"lift_point\": \"[0.0584, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a new nail polish\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3815, 0.3966]\", \"lift_point\": \"[0.3815, 0.3966]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3498, 0.5828]\", \"lift_point\": \"[0.3498, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0584, 0.6103]\", \"lift_point\": \"[0.0584, 0.6103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a new nail polish\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3815, 0.3966]\", \"lift_point\": \"[0.3815, 0.3966]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75185_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75217_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.5999]\", \"lift_point\": \"[0.3442, 0.5999]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75217_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.5999]\", \"lift_point\": \"[0.3442, 0.5999]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.378]\", \"lift_point\": \"[0.0706, 0.378]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75217_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.5999]\", \"lift_point\": \"[0.3442, 0.5999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.378]\", \"lift_point\": \"[0.0706, 0.378]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Germany\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75217_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.5999]\", \"lift_point\": \"[0.3442, 0.5999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.378]\", \"lift_point\": \"[0.0706, 0.378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Germany\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75217_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3442, 0.5999]\", \"lift_point\": \"[0.3442, 0.5999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0706, 0.378]\", \"lift_point\": \"[0.0706, 0.378]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Germany\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Germany?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75217_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8096, 0.6079]\", \"lift_point\": \"[0.8096, 0.6079]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75243_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8096, 0.6079]\", \"lift_point\": \"[0.8096, 0.6079]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.3605]\", \"lift_point\": \"[0.0641, 0.3605]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75243_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8096, 0.6079]\", \"lift_point\": \"[0.8096, 0.6079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.3605]\", \"lift_point\": \"[0.0641, 0.3605]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Venezuela\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75243_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8096, 0.6079]\", \"lift_point\": \"[0.8096, 0.6079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.3605]\", \"lift_point\": \"[0.0641, 0.3605]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Venezuela\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75243_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8096, 0.6079]\", \"lift_point\": \"[0.8096, 0.6079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0641, 0.3605]\", \"lift_point\": \"[0.0641, 0.3605]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in Venezuela\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Venezuela?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75243_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75271_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8184, 0.6118]\", \"lift_point\": \"[0.8184, 0.6118]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75271_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8184, 0.6118]\", \"lift_point\": \"[0.8184, 0.6118]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.5541]\", \"lift_point\": \"[0.0714, 0.5541]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75271_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8184, 0.6118]\", \"lift_point\": \"[0.8184, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.5541]\", \"lift_point\": \"[0.0714, 0.5541]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75271_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8184, 0.6118]\", \"lift_point\": \"[0.8184, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.5541]\", \"lift_point\": \"[0.0714, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75271_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8184, 0.6118]\", \"lift_point\": \"[0.8184, 0.6118]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0714, 0.5541]\", \"lift_point\": \"[0.0714, 0.5541]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75271_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75921_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75921_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6269, 0.3713]\", \"lift_point\": \"[0.6269, 0.3713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75921_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6269, 0.3713]\", \"lift_point\": \"[0.6269, 0.3713]\", \"typed_text\": \"\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6354, 0.3561]\", \"lift_point\": \"[0.6354, 0.3561]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75921_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6269, 0.3713]\", \"lift_point\": \"[0.6269, 0.3713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6354, 0.3561]\", \"lift_point\": \"[0.6354, 0.3561]\", \"typed_text\": \"\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6378, 0.3561]\", \"lift_point\": \"[0.6378, 0.3561]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75921_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6269, 0.3713]\", \"lift_point\": \"[0.6269, 0.3713]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6354, 0.3561]\", \"lift_point\": \"[0.6354, 0.3561]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6378, 0.3561]\", \"lift_point\": \"[0.6378, 0.3561]\", \"typed_text\": \"\" Goal: Search for good BBQ restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/75921_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5887, 0.3817]\", \"lift_point\": \"[0.5887, 0.3817]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2844, 0.3641]\", \"lift_point\": \"[0.2844, 0.3641]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9037]\", \"lift_point\": \"[0.0633, 0.9037]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0459, 0.5181]\", \"lift_point\": \"[0.0459, 0.5181]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Los Angeles \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.3095]\", \"lift_point\": \"[0.1281, 0.3095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3041, 0.3748]\", \"lift_point\": \"[0.3041, 0.3748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2588, 0.8913]\", \"lift_point\": \"[0.2588, 0.8913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.64, 0.3694]\", \"lift_point\": \"[0.64, 0.3694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.38, 0.3022]\", \"lift_point\": \"[0.38, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.64, 0.3694]\", \"lift_point\": \"[0.64, 0.3694]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7376, 0.3849]\", \"lift_point\": \"[0.7376, 0.3849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5555, 0.9799]\", \"lift_point\": \"[0.5555, 0.9799]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.64, 0.3694]\", \"lift_point\": \"[0.64, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7376, 0.3849]\", \"lift_point\": \"[0.7376, 0.3849]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.9414]\", \"lift_point\": \"[0.8352, 0.9414]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1394, 0.9532]\", \"lift_point\": \"[0.1394, 0.9532]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.64, 0.3694]\", \"lift_point\": \"[0.64, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7376, 0.3849]\", \"lift_point\": \"[0.7376, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.9414]\", \"lift_point\": \"[0.8352, 0.9414]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8986, 0.5991]\", \"lift_point\": \"[0.8986, 0.5991]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.5446]\", \"lift_point\": \"[0.7909, 0.5446]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.64, 0.3694]\", \"lift_point\": \"[0.64, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7376, 0.3849]\", \"lift_point\": \"[0.7376, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.9414]\", \"lift_point\": \"[0.8352, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8986, 0.5991]\", \"lift_point\": \"[0.8986, 0.5991]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7462, 0.5963]\", \"lift_point\": \"[0.7462, 0.5963]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1491, 0.9393]\", \"lift_point\": \"[0.1491, 0.9393]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.64, 0.3694]\", \"lift_point\": \"[0.64, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7376, 0.3849]\", \"lift_point\": \"[0.7376, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.9414]\", \"lift_point\": \"[0.8352, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8986, 0.5991]\", \"lift_point\": \"[0.8986, 0.5991]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7462, 0.5963]\", \"lift_point\": \"[0.7462, 0.5963]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6579, 0.9357]\", \"lift_point\": \"[0.6579, 0.9357]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.64, 0.3694]\", \"lift_point\": \"[0.64, 0.3694]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7376, 0.3849]\", \"lift_point\": \"[0.7376, 0.3849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8352, 0.9414]\", \"lift_point\": \"[0.8352, 0.9414]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8986, 0.5991]\", \"lift_point\": \"[0.8986, 0.5991]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7462, 0.5963]\", \"lift_point\": \"[0.7462, 0.5963]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6579, 0.9357]\", \"lift_point\": \"[0.6579, 0.9357]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76074_26.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.4279]\", \"lift_point\": \"[0.1386, 0.4279]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.4279]\", \"lift_point\": \"[0.1386, 0.4279]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3055, 0.4359]\", \"lift_point\": \"[0.3055, 0.4359]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.4279]\", \"lift_point\": \"[0.1386, 0.4279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3055, 0.4359]\", \"lift_point\": \"[0.3055, 0.4359]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.592, 0.4213]\", \"lift_point\": \"[0.592, 0.4213]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.4279]\", \"lift_point\": \"[0.1386, 0.4279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3055, 0.4359]\", \"lift_point\": \"[0.3055, 0.4359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2848, 0.4712]\", \"lift_point\": \"[0.2848, 0.4712]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.4279]\", \"lift_point\": \"[0.1386, 0.4279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3055, 0.4359]\", \"lift_point\": \"[0.3055, 0.4359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0554, 0.9111]\", \"lift_point\": \"[0.0554, 0.9111]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.4279]\", \"lift_point\": \"[0.1386, 0.4279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3055, 0.4359]\", \"lift_point\": \"[0.3055, 0.4359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8072, 0.4337]\", \"lift_point\": \"[0.8072, 0.4337]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0618, 0.3088]\", \"lift_point\": \"[0.0618, 0.3088]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Miami? \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1386, 0.4279]\", \"lift_point\": \"[0.1386, 0.4279]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3055, 0.4359]\", \"lift_point\": \"[0.3055, 0.4359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8072, 0.4337]\", \"lift_point\": \"[0.8072, 0.4337]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Miami?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76164_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1509, 0.2833]\", \"lift_point\": \"[0.1509, 0.2833]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1509, 0.2833]\", \"lift_point\": \"[0.1509, 0.2833]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9171, 0.6295]\", \"lift_point\": \"[0.9171, 0.6295]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1509, 0.2833]\", \"lift_point\": \"[0.1509, 0.2833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9171, 0.6295]\", \"lift_point\": \"[0.9171, 0.6295]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9324, 0.5184]\", \"lift_point\": \"[0.9324, 0.5184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1509, 0.2833]\", \"lift_point\": \"[0.1509, 0.2833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9171, 0.6295]\", \"lift_point\": \"[0.9171, 0.6295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9324, 0.5184]\", \"lift_point\": \"[0.9324, 0.5184]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5279, 0.7919]\", \"lift_point\": \"[0.5279, 0.7919]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0293, 0.6292]\", \"lift_point\": \"[0.0293, 0.6292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5576, 0.6392]\", \"lift_point\": \"[0.5576, 0.6392]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8956, 0.3634]\", \"lift_point\": \"[0.8956, 0.3634]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1509, 0.2833]\", \"lift_point\": \"[0.1509, 0.2833]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9171, 0.6295]\", \"lift_point\": \"[0.9171, 0.6295]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9324, 0.5184]\", \"lift_point\": \"[0.9324, 0.5184]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5279, 0.7919]\", \"lift_point\": \"[0.5279, 0.7919]\", \"typed_text\": \"\" Goal: Check my gmail>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76215_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76280_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.4093]\", \"lift_point\": \"[0.8935, 0.4093]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76280_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.4093]\", \"lift_point\": \"[0.8935, 0.4093]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.2103]\", \"lift_point\": \"[0.0481, 0.2103]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76280_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.4093]\", \"lift_point\": \"[0.8935, 0.4093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.2103]\", \"lift_point\": \"[0.0481, 0.2103]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Atlanta?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76280_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8935, 0.4093]\", \"lift_point\": \"[0.8935, 0.4093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0481, 0.2103]\", \"lift_point\": \"[0.0481, 0.2103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Atlanta?\" Goal: How much does a 3 bedroom apartment rent for in Atlanta?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76280_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Go to Reddit>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76300_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4114]\", \"lift_point\": \"[0.8903, 0.4114]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76300_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4114]\", \"lift_point\": \"[0.8903, 0.4114]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.3729]\", \"lift_point\": \"[0.0852, 0.3729]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76300_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4114]\", \"lift_point\": \"[0.8903, 0.4114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.3729]\", \"lift_point\": \"[0.0852, 0.3729]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76300_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4114]\", \"lift_point\": \"[0.8903, 0.4114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.3729]\", \"lift_point\": \"[0.0852, 0.3729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.3313]\", \"lift_point\": \"[0.1092, 0.3313]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76300_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8903, 0.4114]\", \"lift_point\": \"[0.8903, 0.4114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0852, 0.3729]\", \"lift_point\": \"[0.0852, 0.3729]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1092, 0.3313]\", \"lift_point\": \"[0.1092, 0.3313]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76300_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76354_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3259]\", \"lift_point\": \"[0.8865, 0.3259]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76354_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3259]\", \"lift_point\": \"[0.8865, 0.3259]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.354]\", \"lift_point\": \"[0.0592, 0.354]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76354_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3259]\", \"lift_point\": \"[0.8865, 0.3259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.354]\", \"lift_point\": \"[0.0592, 0.354]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in San Francisco?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76354_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3259]\", \"lift_point\": \"[0.8865, 0.3259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.354]\", \"lift_point\": \"[0.0592, 0.354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in San Francisco?\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1369, 0.4636]\", \"lift_point\": \"[0.1369, 0.4636]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76354_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3259]\", \"lift_point\": \"[0.8865, 0.3259]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0592, 0.354]\", \"lift_point\": \"[0.0592, 0.354]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in San Francisco?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1369, 0.4636]\", \"lift_point\": \"[0.1369, 0.4636]\", \"typed_text\": \"\" Goal: What's the time in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76354_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7998, 0.6996]\", \"lift_point\": \"[0.7998, 0.6996]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4961, 0.5053]\", \"lift_point\": \"[0.4961, 0.5053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1869, 0.3005]\", \"lift_point\": \"[0.1869, 0.3005]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.061, 0.053]\", \"lift_point\": \"[0.061, 0.053]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5008, 0.4922]\", \"lift_point\": \"[0.5008, 0.4922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.0857]\", \"lift_point\": \"[0.8925, 0.0857]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.924, 0.5731]\", \"lift_point\": \"[0.924, 0.5731]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.0857]\", \"lift_point\": \"[0.8925, 0.0857]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3395, 0.4701]\", \"lift_point\": \"[0.3395, 0.4701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.0857]\", \"lift_point\": \"[0.8925, 0.0857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7965, 0.0738]\", \"lift_point\": \"[0.7965, 0.0738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9149, 0.5732]\", \"lift_point\": \"[0.9149, 0.5732]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.0857]\", \"lift_point\": \"[0.8925, 0.0857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7965, 0.0738]\", \"lift_point\": \"[0.7965, 0.0738]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.527, 0.8599]\", \"lift_point\": \"[0.527, 0.8599]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.0857]\", \"lift_point\": \"[0.8925, 0.0857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7965, 0.0738]\", \"lift_point\": \"[0.7965, 0.0738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2678, 0.0983]\", \"lift_point\": \"[0.2678, 0.0983]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6778, 0.0722]\", \"lift_point\": \"[0.6778, 0.0722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8925, 0.0857]\", \"lift_point\": \"[0.8925, 0.0857]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7965, 0.0738]\", \"lift_point\": \"[0.7965, 0.0738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2678, 0.0983]\", \"lift_point\": \"[0.2678, 0.0983]\", \"typed_text\": \"\" Goal: Check my email>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76490_22.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9087, 0.4727]\", \"lift_point\": \"[0.9087, 0.4727]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9087, 0.4727]\", \"lift_point\": \"[0.9087, 0.4727]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.1246]\", \"lift_point\": \"[0.895, 0.1246]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9087, 0.4727]\", \"lift_point\": \"[0.9087, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.1246]\", \"lift_point\": \"[0.895, 0.1246]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2872, 0.2132]\", \"lift_point\": \"[0.2872, 0.2132]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9087, 0.4727]\", \"lift_point\": \"[0.9087, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.1246]\", \"lift_point\": \"[0.895, 0.1246]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2872, 0.2132]\", \"lift_point\": \"[0.2872, 0.2132]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Spain\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9087, 0.4727]\", \"lift_point\": \"[0.9087, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.1246]\", \"lift_point\": \"[0.895, 0.1246]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2872, 0.2132]\", \"lift_point\": \"[0.2872, 0.2132]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Spain\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [PRESS_ENTER,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9087, 0.4727]\", \"lift_point\": \"[0.9087, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.1246]\", \"lift_point\": \"[0.895, 0.1246]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2872, 0.2132]\", \"lift_point\": \"[0.2872, 0.2132]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Spain\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.805, 0.7028]\", \"lift_point\": \"[0.805, 0.7028]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9087, 0.4727]\", \"lift_point\": \"[0.9087, 0.4727]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.895, 0.1246]\", \"lift_point\": \"[0.895, 0.1246]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2872, 0.2132]\", \"lift_point\": \"[0.2872, 0.2132]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Spain\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Spain?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76733_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6947]\", \"lift_point\": \"[0.7931, 0.6947]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6947]\", \"lift_point\": \"[0.7931, 0.6947]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3791]\", \"lift_point\": \"[0.0582, 0.3791]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6947]\", \"lift_point\": \"[0.7931, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3791]\", \"lift_point\": \"[0.0582, 0.3791]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6947]\", \"lift_point\": \"[0.7931, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3791]\", \"lift_point\": \"[0.0582, 0.3791]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6947]\", \"lift_point\": \"[0.7931, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3791]\", \"lift_point\": \"[0.0582, 0.3791]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6947]\", \"lift_point\": \"[0.7931, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3791]\", \"lift_point\": \"[0.0582, 0.3791]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5256, 0.4326]\", \"lift_point\": \"[0.5256, 0.4326]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7931, 0.6947]\", \"lift_point\": \"[0.7931, 0.6947]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0582, 0.3791]\", \"lift_point\": \"[0.0582, 0.3791]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good Italian restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5256, 0.4326]\", \"lift_point\": \"[0.5256, 0.4326]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76774_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What are the best selling refrigerators at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76909_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3451, 0.4831]\", \"lift_point\": \"[0.3451, 0.4831]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76909_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3451, 0.4831]\", \"lift_point\": \"[0.3451, 0.4831]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Lowes?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.6439]\", \"lift_point\": \"[0.0653, 0.6439]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76909_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3451, 0.4831]\", \"lift_point\": \"[0.3451, 0.4831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.6439]\", \"lift_point\": \"[0.0653, 0.6439]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Lowes?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Lowes?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76909_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3451, 0.4831]\", \"lift_point\": \"[0.3451, 0.4831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.6439]\", \"lift_point\": \"[0.0653, 0.6439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Lowes?\" Goal: What are the best selling refrigerators at Lowes?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76909_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3451, 0.4831]\", \"lift_point\": \"[0.3451, 0.4831]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.6439]\", \"lift_point\": \"[0.0653, 0.6439]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the best selling refrigerators at Lowes?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What are the best selling refrigerators at Lowes?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76909_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76940_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6864]\", \"lift_point\": \"[0.6352, 0.6864]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76940_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6864]\", \"lift_point\": \"[0.6352, 0.6864]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76940_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6864]\", \"lift_point\": \"[0.6352, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.5767]\", \"lift_point\": \"[0.0649, 0.5767]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76940_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6864]\", \"lift_point\": \"[0.6352, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.5767]\", \"lift_point\": \"[0.0649, 0.5767]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76940_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6864]\", \"lift_point\": \"[0.6352, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.5767]\", \"lift_point\": \"[0.0649, 0.5767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit app\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76940_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6352, 0.6864]\", \"lift_point\": \"[0.6352, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.5767]\", \"lift_point\": \"[0.0649, 0.5767]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Reddit app\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Reddit app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76940_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3702, 0.4935]\", \"lift_point\": \"[0.3702, 0.4935]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3702, 0.4935]\", \"lift_point\": \"[0.3702, 0.4935]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.5428]\", \"lift_point\": \"[0.0562, 0.5428]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3702, 0.4935]\", \"lift_point\": \"[0.3702, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.5428]\", \"lift_point\": \"[0.0562, 0.5428]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3702, 0.4935]\", \"lift_point\": \"[0.3702, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.5428]\", \"lift_point\": \"[0.0562, 0.5428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3702, 0.4935]\", \"lift_point\": \"[0.3702, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.5428]\", \"lift_point\": \"[0.0562, 0.5428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3563, 0.0811]\", \"lift_point\": \"[0.3563, 0.0811]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3725, 0.5031]\", \"lift_point\": \"[0.3725, 0.5031]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3702, 0.4935]\", \"lift_point\": \"[0.3702, 0.4935]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0562, 0.5428]\", \"lift_point\": \"[0.0562, 0.5428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3563, 0.0811]\", \"lift_point\": \"[0.3563, 0.0811]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/76978_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lights from Barcelona to Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lights from Barcelona to Mexico city\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lights from Barcelona to Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.4193]\", \"lift_point\": \"[0.1537, 0.4193]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lights from Barcelona to Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.4193]\", \"lift_point\": \"[0.1537, 0.4193]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5051, 0.6922]\", \"lift_point\": \"[0.5051, 0.6922]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lights from Barcelona to Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.4193]\", \"lift_point\": \"[0.1537, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lights from Barcelona to Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.4193]\", \"lift_point\": \"[0.1537, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2133, 0.2494]\", \"lift_point\": \"[0.2133, 0.2494]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.693]\", \"lift_point\": \"[0.1412, 0.693]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.8872]\", \"lift_point\": \"[0.073, 0.8872]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lights from Barcelona to Mexico city\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.4193]\", \"lift_point\": \"[0.1537, 0.4193]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2133, 0.2494]\", \"lift_point\": \"[0.2133, 0.2494]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77185_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.6952]\", \"lift_point\": \"[0.4698, 0.6952]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.6952]\", \"lift_point\": \"[0.4698, 0.6952]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2484, 0.3022]\", \"lift_point\": \"[0.2484, 0.3022]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.6952]\", \"lift_point\": \"[0.4698, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2484, 0.3022]\", \"lift_point\": \"[0.2484, 0.3022]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.6952]\", \"lift_point\": \"[0.4698, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2484, 0.3022]\", \"lift_point\": \"[0.2484, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.6952]\", \"lift_point\": \"[0.4698, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2484, 0.3022]\", \"lift_point\": \"[0.2484, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1455, 0.4439]\", \"lift_point\": \"[0.1455, 0.4439]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4698, 0.6952]\", \"lift_point\": \"[0.4698, 0.6952]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2484, 0.3022]\", \"lift_point\": \"[0.2484, 0.3022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Norway?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1455, 0.4439]\", \"lift_point\": \"[0.1455, 0.4439]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77237_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.4575]\", \"lift_point\": \"[0.0608, 0.4575]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.4575]\", \"lift_point\": \"[0.0608, 0.4575]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Las Vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.4575]\", \"lift_point\": \"[0.0608, 0.4575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Las Vegas\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.4855]\", \"lift_point\": \"[0.0632, 0.4855]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.4575]\", \"lift_point\": \"[0.0608, 0.4575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.4855]\", \"lift_point\": \"[0.0632, 0.4855]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1946, 0.4792]\", \"lift_point\": \"[0.1946, 0.4792]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.4575]\", \"lift_point\": \"[0.0608, 0.4575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.4855]\", \"lift_point\": \"[0.0632, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1946, 0.4792]\", \"lift_point\": \"[0.1946, 0.4792]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1981, 0.4148]\", \"lift_point\": \"[0.1981, 0.4148]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.4575]\", \"lift_point\": \"[0.0608, 0.4575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.4855]\", \"lift_point\": \"[0.0632, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1946, 0.4792]\", \"lift_point\": \"[0.1946, 0.4792]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1981, 0.4148]\", \"lift_point\": \"[0.1981, 0.4148]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4365, 0.2684]\", \"lift_point\": \"[0.4365, 0.2684]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7988, 0.6964]\", \"lift_point\": \"[0.7988, 0.6964]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0608, 0.4575]\", \"lift_point\": \"[0.0608, 0.4575]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0632, 0.4855]\", \"lift_point\": \"[0.0632, 0.4855]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1946, 0.4792]\", \"lift_point\": \"[0.1946, 0.4792]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1981, 0.4148]\", \"lift_point\": \"[0.1981, 0.4148]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4365, 0.2684]\", \"lift_point\": \"[0.4365, 0.2684]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77289_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4986]\", \"lift_point\": \"[0.0596, 0.4986]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4986]\", \"lift_point\": \"[0.0596, 0.4986]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.4861]\", \"lift_point\": \"[0.0609, 0.4861]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4986]\", \"lift_point\": \"[0.0596, 0.4986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.4861]\", \"lift_point\": \"[0.0609, 0.4861]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0493, 0.5165]\", \"lift_point\": \"[0.0493, 0.5165]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4986]\", \"lift_point\": \"[0.0596, 0.4986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.4861]\", \"lift_point\": \"[0.0609, 0.4861]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0493, 0.5165]\", \"lift_point\": \"[0.0493, 0.5165]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"chipotle ?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4986]\", \"lift_point\": \"[0.0596, 0.4986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.4861]\", \"lift_point\": \"[0.0609, 0.4861]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0493, 0.5165]\", \"lift_point\": \"[0.0493, 0.5165]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"chipotle?\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1259, 0.3229]\", \"lift_point\": \"[0.1259, 0.3229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.1352]\", \"lift_point\": \"[0.7928, 0.1352]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0725, 0.4733]\", \"lift_point\": \"[0.0725, 0.4733]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest Chipotle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.4986]\", \"lift_point\": \"[0.0596, 0.4986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0609, 0.4861]\", \"lift_point\": \"[0.0609, 0.4861]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0493, 0.5165]\", \"lift_point\": \"[0.0493, 0.5165]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"chipotle?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1259, 0.3229]\", \"lift_point\": \"[0.1259, 0.3229]\", \"typed_text\": \"\" Goal: How do I get to the nearest Chipotle?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77309_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Sydney>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.24, 0.6827]\", \"lift_point\": \"[0.24, 0.6827]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77314_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.24, 0.6827]\", \"lift_point\": \"[0.24, 0.6827]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Sydney>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Sydney\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77314_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.24, 0.6827]\", \"lift_point\": \"[0.24, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Sydney\" Goal: Search for flights from NYC to Sydney>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.384]\", \"lift_point\": \"[0.1297, 0.384]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77314_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.24, 0.6827]\", \"lift_point\": \"[0.24, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.384]\", \"lift_point\": \"[0.1297, 0.384]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Sydney>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7676, 0.3441]\", \"lift_point\": \"[0.7676, 0.3441]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77314_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.24, 0.6827]\", \"lift_point\": \"[0.24, 0.6827]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"flights from NYC to Sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1297, 0.384]\", \"lift_point\": \"[0.1297, 0.384]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7676, 0.3441]\", \"lift_point\": \"[0.7676, 0.3441]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Sydney>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77314_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2362, 0.6956]\", \"lift_point\": \"[0.2362, 0.6956]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2362, 0.6956]\", \"lift_point\": \"[0.2362, 0.6956]\", \"typed_text\": \"\" Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.722]\", \"lift_point\": \"[0.0616, 0.722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2362, 0.6956]\", \"lift_point\": \"[0.2362, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.722]\", \"lift_point\": \"[0.0616, 0.722]\", \"typed_text\": \"\" Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"singapore\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2362, 0.6956]\", \"lift_point\": \"[0.2362, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.722]\", \"lift_point\": \"[0.0616, 0.722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"singapore\" Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1323, 0.2234]\", \"lift_point\": \"[0.1323, 0.2234]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2362, 0.6956]\", \"lift_point\": \"[0.2362, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.722]\", \"lift_point\": \"[0.0616, 0.722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"singapore\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1323, 0.2234]\", \"lift_point\": \"[0.1323, 0.2234]\", \"typed_text\": \"\" Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2364, 0.3074]\", \"lift_point\": \"[0.2364, 0.3074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2362, 0.6956]\", \"lift_point\": \"[0.2362, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.722]\", \"lift_point\": \"[0.0616, 0.722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"singapore\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1323, 0.2234]\", \"lift_point\": \"[0.1323, 0.2234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2364, 0.3074]\", \"lift_point\": \"[0.2364, 0.3074]\", \"typed_text\": \"\" Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3413, 0.2505]\", \"lift_point\": \"[0.3413, 0.2505]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2362, 0.6956]\", \"lift_point\": \"[0.2362, 0.6956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0616, 0.722]\", \"lift_point\": \"[0.0616, 0.722]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"singapore\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1323, 0.2234]\", \"lift_point\": \"[0.1323, 0.2234]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2364, 0.3074]\", \"lift_point\": \"[0.2364, 0.3074]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3413, 0.2505]\", \"lift_point\": \"[0.3413, 0.2505]\", \"typed_text\": \"\" Goal: What's the news in Singapore?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77512_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2354, 0.6773]\", \"lift_point\": \"[0.2354, 0.6773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0572, 0.4785]\", \"lift_point\": \"[0.0572, 0.4785]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"rent for 3 bedroom apartment in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0627, 0.5717]\", \"lift_point\": \"[0.0627, 0.5717]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" in chicago\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1198, 0.5591]\", \"lift_point\": \"[0.1198, 0.5591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3566, 0.2985]\", \"lift_point\": \"[0.3566, 0.2985]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2859, 0.2866]\", \"lift_point\": \"[0.2859, 0.2866]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6261, 0.3049]\", \"lift_point\": \"[0.6261, 0.3049]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7609, 0.4401]\", \"lift_point\": \"[0.7609, 0.4401]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7609, 0.4401]\", \"lift_point\": \"[0.7609, 0.4401]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1846, 0.8803]\", \"lift_point\": \"[0.1846, 0.8803]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7609, 0.4401]\", \"lift_point\": \"[0.7609, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1846, 0.8803]\", \"lift_point\": \"[0.1846, 0.8803]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.551, 0.0637]\", \"lift_point\": \"[0.551, 0.0637]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7609, 0.4401]\", \"lift_point\": \"[0.7609, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1846, 0.8803]\", \"lift_point\": \"[0.1846, 0.8803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.551, 0.0637]\", \"lift_point\": \"[0.551, 0.0637]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8982, 0.7229]\", \"lift_point\": \"[0.8982, 0.7229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7609, 0.4401]\", \"lift_point\": \"[0.7609, 0.4401]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1846, 0.8803]\", \"lift_point\": \"[0.1846, 0.8803]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.551, 0.0637]\", \"lift_point\": \"[0.551, 0.0637]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8982, 0.7229]\", \"lift_point\": \"[0.8982, 0.7229]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77654_32.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\" Goal: How big is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\" Goal: How big is the earth?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how big is the earth\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how big is the earth\" Goal: How big is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.2182]\", \"lift_point\": \"[0.1419, 0.2182]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how big is the earth\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.2182]\", \"lift_point\": \"[0.1419, 0.2182]\", \"typed_text\": \"\" Goal: How big is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.2131]\", \"lift_point\": \"[0.1368, 0.2131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how big is the earth\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.2182]\", \"lift_point\": \"[0.1419, 0.2182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.2131]\", \"lift_point\": \"[0.1368, 0.2131]\", \"typed_text\": \"\" Goal: How big is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3915, 0.4246]\", \"lift_point\": \"[0.3915, 0.4246]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how big is the earth\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.2182]\", \"lift_point\": \"[0.1419, 0.2182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.2131]\", \"lift_point\": \"[0.1368, 0.2131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3915, 0.4246]\", \"lift_point\": \"[0.3915, 0.4246]\", \"typed_text\": \"\" Goal: How big is the earth?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3296, 0.4224]\", \"lift_point\": \"[0.3296, 0.4224]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.6619]\", \"lift_point\": \"[0.2339, 0.6619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.064, 0.3033]\", \"lift_point\": \"[0.064, 0.3033]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how big is the earth\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1419, 0.2182]\", \"lift_point\": \"[0.1419, 0.2182]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1368, 0.2131]\", \"lift_point\": \"[0.1368, 0.2131]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3915, 0.4246]\", \"lift_point\": \"[0.3915, 0.4246]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3296, 0.4224]\", \"lift_point\": \"[0.3296, 0.4224]\", \"typed_text\": \"\" Goal: How big is the earth?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77719_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sydney\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sydney\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.3759]\", \"lift_point\": \"[0.1537, 0.3759]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.3759]\", \"lift_point\": \"[0.1537, 0.3759]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5208, 0.8454]\", \"lift_point\": \"[0.5208, 0.8454]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.3759]\", \"lift_point\": \"[0.1537, 0.3759]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5208, 0.8454]\", \"lift_point\": \"[0.5208, 0.8454]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.3257]\", \"lift_point\": \"[0.0623, 0.3257]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.3759]\", \"lift_point\": \"[0.1537, 0.3759]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5208, 0.8454]\", \"lift_point\": \"[0.5208, 0.8454]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.3257]\", \"lift_point\": \"[0.0623, 0.3257]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.3759]\", \"lift_point\": \"[0.1537, 0.3759]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5208, 0.8454]\", \"lift_point\": \"[0.5208, 0.8454]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.3257]\", \"lift_point\": \"[0.0623, 0.3257]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1578, 0.3203]\", \"lift_point\": \"[0.1578, 0.3203]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8864, 0.3751]\", \"lift_point\": \"[0.8864, 0.3751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0382, 0.3047]\", \"lift_point\": \"[0.0382, 0.3047]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sydney\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1537, 0.3759]\", \"lift_point\": \"[0.1537, 0.3759]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5208, 0.8454]\", \"lift_point\": \"[0.5208, 0.8454]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.3257]\", \"lift_point\": \"[0.0623, 0.3257]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" weather\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1578, 0.3203]\", \"lift_point\": \"[0.1578, 0.3203]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77763_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.5285]\", \"lift_point\": \"[0.8704, 0.5285]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77796_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.5285]\", \"lift_point\": \"[0.8704, 0.5285]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77796_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.5285]\", \"lift_point\": \"[0.8704, 0.5285]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6704]\", \"lift_point\": \"[0.7825, 0.6704]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77796_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.5285]\", \"lift_point\": \"[0.8704, 0.5285]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6704]\", \"lift_point\": \"[0.7825, 0.6704]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.8195]\", \"lift_point\": \"[0.0727, 0.8195]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77796_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.5285]\", \"lift_point\": \"[0.8704, 0.5285]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6704]\", \"lift_point\": \"[0.7825, 0.6704]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.8195]\", \"lift_point\": \"[0.0727, 0.8195]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.0733]\", \"lift_point\": \"[0.0705, 0.0733]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77796_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8704, 0.5285]\", \"lift_point\": \"[0.8704, 0.5285]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6704]\", \"lift_point\": \"[0.7825, 0.6704]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.8195]\", \"lift_point\": \"[0.0727, 0.8195]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0705, 0.0733]\", \"lift_point\": \"[0.0705, 0.0733]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77796_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3928]\", \"lift_point\": \"[0.134, 0.3928]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3928]\", \"lift_point\": \"[0.134, 0.3928]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.289, 0.4324]\", \"lift_point\": \"[0.289, 0.4324]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3928]\", \"lift_point\": \"[0.134, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.289, 0.4324]\", \"lift_point\": \"[0.289, 0.4324]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9038, 0.5987]\", \"lift_point\": \"[0.9038, 0.5987]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3928]\", \"lift_point\": \"[0.134, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.289, 0.4324]\", \"lift_point\": \"[0.289, 0.4324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9038, 0.5987]\", \"lift_point\": \"[0.9038, 0.5987]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3928]\", \"lift_point\": \"[0.134, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.289, 0.4324]\", \"lift_point\": \"[0.289, 0.4324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9038, 0.5987]\", \"lift_point\": \"[0.9038, 0.5987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8856, 0.5887]\", \"lift_point\": \"[0.8856, 0.5887]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.881, 0.4442]\", \"lift_point\": \"[0.881, 0.4442]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Los Angeles\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.134, 0.3928]\", \"lift_point\": \"[0.134, 0.3928]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.289, 0.4324]\", \"lift_point\": \"[0.289, 0.4324]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9038, 0.5987]\", \"lift_point\": \"[0.9038, 0.5987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8856, 0.5887]\", \"lift_point\": \"[0.8856, 0.5887]\", \"typed_text\": \"\" Goal: Search for hotels in Los Angeles>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77846_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants on Maps\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants on Maps\" Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.8901]\", \"lift_point\": \"[0.0689, 0.8901]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants on Maps\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.8901]\", \"lift_point\": \"[0.0689, 0.8901]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4391, 0.8688]\", \"lift_point\": \"[0.4391, 0.8688]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants on Maps\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.8901]\", \"lift_point\": \"[0.0689, 0.8901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4391, 0.8688]\", \"lift_point\": \"[0.4391, 0.8688]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.3608]\", \"lift_point\": \"[0.0686, 0.3608]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants on Maps\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.8901]\", \"lift_point\": \"[0.0689, 0.8901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4391, 0.8688]\", \"lift_point\": \"[0.4391, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.3608]\", \"lift_point\": \"[0.0686, 0.3608]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants on Maps\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.8901]\", \"lift_point\": \"[0.0689, 0.8901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4391, 0.8688]\", \"lift_point\": \"[0.4391, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.3608]\", \"lift_point\": \"[0.0686, 0.3608]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants \" Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.4987]\", \"lift_point\": \"[0.1434, 0.4987]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8807, 0.5032]\", \"lift_point\": \"[0.8807, 0.5032]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants on Maps\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0689, 0.8901]\", \"lift_point\": \"[0.0689, 0.8901]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4391, 0.8688]\", \"lift_point\": \"[0.4391, 0.8688]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0686, 0.3608]\", \"lift_point\": \"[0.0686, 0.3608]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good Italian restaurants \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1434, 0.4987]\", \"lift_point\": \"[0.1434, 0.4987]\", \"typed_text\": \"\" Goal: Search for good Italian restaurants on Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/77982_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3726]\", \"lift_point\": \"[0.8843, 0.3726]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78005_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3726]\", \"lift_point\": \"[0.8843, 0.3726]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78005_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3726]\", \"lift_point\": \"[0.8843, 0.3726]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1471, 0.2619]\", \"lift_point\": \"[0.1471, 0.2619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78005_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3726]\", \"lift_point\": \"[0.8843, 0.3726]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1471, 0.2619]\", \"lift_point\": \"[0.1471, 0.2619]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1992, 0.1891]\", \"lift_point\": \"[0.1992, 0.1891]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78005_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8843, 0.3726]\", \"lift_point\": \"[0.8843, 0.3726]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Pakistan?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1471, 0.2619]\", \"lift_point\": \"[0.1471, 0.2619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1992, 0.1891]\", \"lift_point\": \"[0.1992, 0.1891]\", \"typed_text\": \"\" Goal: What's the news in Pakistan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78005_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Google Play Music app>55555", + "response": "Action Plan: [PRESS_HOME,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78062_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Google Play Music app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78062_2.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78380_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8794, 0.3933]\", \"lift_point\": \"[0.8794, 0.3933]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78380_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8794, 0.3933]\", \"lift_point\": \"[0.8794, 0.3933]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Sydney?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78380_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8794, 0.3933]\", \"lift_point\": \"[0.8794, 0.3933]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Sydney?\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.3901]\", \"lift_point\": \"[0.1412, 0.3901]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78380_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8794, 0.3933]\", \"lift_point\": \"[0.8794, 0.3933]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Sydney?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1412, 0.3901]\", \"lift_point\": \"[0.1412, 0.3901]\", \"typed_text\": \"\" Goal: What's the weather like in Sydney?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78380_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78386_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.296]\", \"lift_point\": \"[0.8949, 0.296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78386_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.296]\", \"lift_point\": \"[0.8949, 0.296]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.3449]\", \"lift_point\": \"[0.8777, 0.3449]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78386_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.296]\", \"lift_point\": \"[0.8949, 0.296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.3449]\", \"lift_point\": \"[0.8777, 0.3449]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation product\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78386_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.296]\", \"lift_point\": \"[0.8949, 0.296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.3449]\", \"lift_point\": \"[0.8777, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation product\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.5548]\", \"lift_point\": \"[0.1477, 0.5548]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78386_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8949, 0.296]\", \"lift_point\": \"[0.8949, 0.296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.3449]\", \"lift_point\": \"[0.8777, 0.3449]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new foundation product\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.5548]\", \"lift_point\": \"[0.1477, 0.5548]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78386_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.2806]\", \"lift_point\": \"[0.8777, 0.2806]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.2806]\", \"lift_point\": \"[0.8777, 0.2806]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.3423]\", \"lift_point\": \"[0.8833, 0.3423]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.2806]\", \"lift_point\": \"[0.8777, 0.2806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.3423]\", \"lift_point\": \"[0.8833, 0.3423]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8972, 0.6467]\", \"lift_point\": \"[0.8972, 0.6467]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.2806]\", \"lift_point\": \"[0.8777, 0.2806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.3423]\", \"lift_point\": \"[0.8833, 0.3423]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8972, 0.6467]\", \"lift_point\": \"[0.8972, 0.6467]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.2806]\", \"lift_point\": \"[0.8777, 0.2806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.3423]\", \"lift_point\": \"[0.8833, 0.3423]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8972, 0.6467]\", \"lift_point\": \"[0.8972, 0.6467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"esktop\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.2806]\", \"lift_point\": \"[0.8777, 0.2806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.3423]\", \"lift_point\": \"[0.8833, 0.3423]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8972, 0.6467]\", \"lift_point\": \"[0.8972, 0.6467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"esktop\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1442, 0.5278]\", \"lift_point\": \"[0.1442, 0.5278]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8777, 0.2806]\", \"lift_point\": \"[0.8777, 0.2806]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8833, 0.3423]\", \"lift_point\": \"[0.8833, 0.3423]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8972, 0.6467]\", \"lift_point\": \"[0.8972, 0.6467]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Show me some nice wallpapers for my desktop\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"esktop\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1442, 0.5278]\", \"lift_point\": \"[0.1442, 0.5278]\", \"typed_text\": \"\" Goal: Show me some nice wallpapers for my desktop>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78401_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8869, 0.6155]\", \"lift_point\": \"[0.8869, 0.6155]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8879, 0.694]\", \"lift_point\": \"[0.8879, 0.694]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"r in denver\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.22, 0.2664]\", \"lift_point\": \"[0.22, 0.2664]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3187, 0.2868]\", \"lift_point\": \"[0.3187, 0.2868]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8804, 0.8006]\", \"lift_point\": \"[0.8804, 0.8006]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7018, 0.8241]\", \"lift_point\": \"[0.7018, 0.8241]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7018, 0.8241]\", \"lift_point\": \"[0.7018, 0.8241]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4877, 0.9055]\", \"lift_point\": \"[0.4877, 0.9055]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7018, 0.8241]\", \"lift_point\": \"[0.7018, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.136, 0.1001]\", \"lift_point\": \"[0.136, 0.1001]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7018, 0.8241]\", \"lift_point\": \"[0.7018, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1358, 0.1133]\", \"lift_point\": \"[0.1358, 0.1133]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7018, 0.8241]\", \"lift_point\": \"[0.7018, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.83]\", \"lift_point\": \"[0.1954, 0.83]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7018, 0.8241]\", \"lift_point\": \"[0.7018, 0.8241]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1954, 0.83]\", \"lift_point\": \"[0.1954, 0.83]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78434_22.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78573_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3838, 0.8511]\", \"lift_point\": \"[0.3838, 0.8511]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78573_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3838, 0.8511]\", \"lift_point\": \"[0.3838, 0.8511]\", \"typed_text\": \"\" Goal: Open the contacts>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78573_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8043, 0.7189]\", \"lift_point\": \"[0.8043, 0.7189]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78663_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8043, 0.7189]\", \"lift_point\": \"[0.8043, 0.7189]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0794, 0.4407]\", \"lift_point\": \"[0.0794, 0.4407]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78663_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8043, 0.7189]\", \"lift_point\": \"[0.8043, 0.7189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0794, 0.4407]\", \"lift_point\": \"[0.0794, 0.4407]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"usd to yuan\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78663_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8043, 0.7189]\", \"lift_point\": \"[0.8043, 0.7189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0794, 0.4407]\", \"lift_point\": \"[0.0794, 0.4407]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"usd to yuan\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1632, 0.4217]\", \"lift_point\": \"[0.1632, 0.4217]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78663_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8043, 0.7189]\", \"lift_point\": \"[0.8043, 0.7189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0794, 0.4407]\", \"lift_point\": \"[0.0794, 0.4407]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"usd to yuan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1632, 0.4217]\", \"lift_point\": \"[0.1632, 0.4217]\", \"typed_text\": \"\" Goal: What's US dollar exchange rate against the Chinese Yuan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78663_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78840_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1536]\", \"lift_point\": \"[0.793, 0.1536]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78840_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.793, 0.1536]\", \"lift_point\": \"[0.793, 0.1536]\", \"typed_text\": \"\" Goal: Go to Reddit>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78840_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Thailand?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78906_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Thailand?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7166]\", \"lift_point\": \"[0.7884, 0.7166]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78906_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7166]\", \"lift_point\": \"[0.7884, 0.7166]\", \"typed_text\": \"\" Goal: What's the news in Thailand?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0889, 0.5468]\", \"lift_point\": \"[0.0889, 0.5468]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78906_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7166]\", \"lift_point\": \"[0.7884, 0.7166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0889, 0.5468]\", \"lift_point\": \"[0.0889, 0.5468]\", \"typed_text\": \"\" Goal: What's the news in Thailand?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in thailand\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78906_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7166]\", \"lift_point\": \"[0.7884, 0.7166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0889, 0.5468]\", \"lift_point\": \"[0.0889, 0.5468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in thailand\" Goal: What's the news in Thailand?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1601, 0.813]\", \"lift_point\": \"[0.1601, 0.813]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78906_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7884, 0.7166]\", \"lift_point\": \"[0.7884, 0.7166]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0889, 0.5468]\", \"lift_point\": \"[0.0889, 0.5468]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in thailand\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1601, 0.813]\", \"lift_point\": \"[0.1601, 0.813]\", \"typed_text\": \"\" Goal: What's the news in Thailand?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/78906_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [PRESS_HOME,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Where can I buy a nice beach towel?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4075, 0.2072]\", \"lift_point\": \"[0.4075, 0.2072]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [PRESS_BACK,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8376, 0.5474]\", \"lift_point\": \"[0.8376, 0.5474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4472, 0.7619]\", \"lift_point\": \"[0.4472, 0.7619]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4026, 0.2483]\", \"lift_point\": \"[0.4026, 0.2483]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3146, 0.2455]\", \"lift_point\": \"[0.3146, 0.2455]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_54.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_55.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_56.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2281, 0.1615]\", \"lift_point\": \"[0.2281, 0.1615]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.846, 0.3522]\", \"lift_point\": \"[0.846, 0.3522]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_57.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8576, 0.701]\", \"lift_point\": \"[0.8576, 0.701]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.846, 0.3522]\", \"lift_point\": \"[0.846, 0.3522]\", \"typed_text\": \"\" Goal: Where can I buy a nice beach towel?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79301_58.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8005, 0.6926]\", \"lift_point\": \"[0.8005, 0.6926]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3471, 0.6703]\", \"lift_point\": \"[0.3471, 0.6703]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8741, 0.2573]\", \"lift_point\": \"[0.8741, 0.2573]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good french restaurants\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5589, 0.0903]\", \"lift_point\": \"[0.5589, 0.0903]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5434, 0.14]\", \"lift_point\": \"[0.5434, 0.14]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5434, 0.14]\", \"lift_point\": \"[0.5434, 0.14]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7193, 0.2381]\", \"lift_point\": \"[0.7193, 0.2381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5434, 0.14]\", \"lift_point\": \"[0.5434, 0.14]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7193, 0.2381]\", \"lift_point\": \"[0.7193, 0.2381]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.3024]\", \"lift_point\": \"[0.7264, 0.3024]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5434, 0.14]\", \"lift_point\": \"[0.5434, 0.14]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7193, 0.2381]\", \"lift_point\": \"[0.7193, 0.2381]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7264, 0.3024]\", \"lift_point\": \"[0.7264, 0.3024]\", \"typed_text\": \"\" Goal: Searchfor good French restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79328_27.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7752, 0.6834]\", \"lift_point\": \"[0.7752, 0.6834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0525, 0.4934]\", \"lift_point\": \"[0.0525, 0.4934]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at Chick-fil-A?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6076, 0.4409]\", \"lift_point\": \"[0.6076, 0.4409]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_20.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_21.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_22.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_23.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_24.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_25.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_26.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_27.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_28.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_29.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_30.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_31.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_32.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_33.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_34.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_35.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_36.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_37.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_38.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_39.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7031, 0.5546]\", \"lift_point\": \"[0.7031, 0.5546]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_40.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_41.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_42.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_43.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_44.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_45.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2804, 0.1649]\", \"lift_point\": \"[0.2804, 0.1649]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_46.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_47.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_48.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7902, 0.7447]\", \"lift_point\": \"[0.7902, 0.7447]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_49.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_50.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_51.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_52.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2558, 0.2172]\", \"lift_point\": \"[0.2558, 0.2172]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_53.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_54.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_55.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_56.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8458, 0.5369]\", \"lift_point\": \"[0.8458, 0.5369]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_57.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5453, 0.7439]\", \"lift_point\": \"[0.5453, 0.7439]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_58.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_59.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_60.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_61.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5107, 0.5786]\", \"lift_point\": \"[0.5107, 0.5786]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_62.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_63.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_64.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_65.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_66.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_67.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_68.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_69.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_70.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_71.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_72.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_73.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_74.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_75.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_76.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_77.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.5123]\", \"lift_point\": \"[0.2339, 0.5123]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_78.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2339, 0.5123]\", \"lift_point\": \"[0.2339, 0.5123]\", \"typed_text\": \"\" Goal: What's on the menu at Chick-fil-A?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79407_79.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79892_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.6176]\", \"lift_point\": \"[0.6333, 0.6176]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79892_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.6176]\", \"lift_point\": \"[0.6333, 0.6176]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.881]\", \"lift_point\": \"[0.1281, 0.881]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79892_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.6176]\", \"lift_point\": \"[0.6333, 0.6176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.881]\", \"lift_point\": \"[0.1281, 0.881]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3572, 0.3254]\", \"lift_point\": \"[0.3572, 0.3254]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79892_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6333, 0.6176]\", \"lift_point\": \"[0.6333, 0.6176]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1281, 0.881]\", \"lift_point\": \"[0.1281, 0.881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3572, 0.3254]\", \"lift_point\": \"[0.3572, 0.3254]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/79892_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Chile>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80037_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Chile>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7773, 0.1975]\", \"lift_point\": \"[0.7773, 0.1975]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80037_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7773, 0.1975]\", \"lift_point\": \"[0.7773, 0.1975]\", \"typed_text\": \"\" Goal: Google the capital of Chile>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of chile\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80037_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7773, 0.1975]\", \"lift_point\": \"[0.7773, 0.1975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of chile\" Goal: Google the capital of Chile>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2623, 0.3865]\", \"lift_point\": \"[0.2623, 0.3865]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80037_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7773, 0.1975]\", \"lift_point\": \"[0.7773, 0.1975]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of chile\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2623, 0.3865]\", \"lift_point\": \"[0.2623, 0.3865]\", \"typed_text\": \"\" Goal: Google the capital of Chile>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80037_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7851, 0.6795]\", \"lift_point\": \"[0.7851, 0.6795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0504, 0.3893]\", \"lift_point\": \"[0.0504, 0.3893]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5417, 0.5971]\", \"lift_point\": \"[0.5417, 0.5971]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0561, 0.9221]\", \"lift_point\": \"[0.0561, 0.9221]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5417, 0.5971]\", \"lift_point\": \"[0.5417, 0.5971]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2171, 0.3683]\", \"lift_point\": \"[0.2171, 0.3683]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5417, 0.5971]\", \"lift_point\": \"[0.5417, 0.5971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2171, 0.3683]\", \"lift_point\": \"[0.2171, 0.3683]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lipstick\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1934, 0.2322]\", \"lift_point\": \"[0.1934, 0.2322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5417, 0.5971]\", \"lift_point\": \"[0.5417, 0.5971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2171, 0.3683]\", \"lift_point\": \"[0.2171, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lipstick\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5417, 0.5971]\", \"lift_point\": \"[0.5417, 0.5971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2171, 0.3683]\", \"lift_point\": \"[0.2171, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lipstick\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.304, 0.3131]\", \"lift_point\": \"[0.304, 0.3131]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6837, 0.165]\", \"lift_point\": \"[0.6837, 0.165]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6967, 0.4492]\", \"lift_point\": \"[0.6967, 0.4492]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2712, 0.3339]\", \"lift_point\": \"[0.2712, 0.3339]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5417, 0.5971]\", \"lift_point\": \"[0.5417, 0.5971]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2171, 0.3683]\", \"lift_point\": \"[0.2171, 0.3683]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"lipstick\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.304, 0.3131]\", \"lift_point\": \"[0.304, 0.3131]\", \"typed_text\": \"\" Goal: Search for a new lipstick on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80102_16.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.687]\", \"lift_point\": \"[0.7928, 0.687]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80107_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.687]\", \"lift_point\": \"[0.7928, 0.687]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.3919]\", \"lift_point\": \"[0.0448, 0.3919]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80107_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.687]\", \"lift_point\": \"[0.7928, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.3919]\", \"lift_point\": \"[0.0448, 0.3919]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of norway?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80107_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.687]\", \"lift_point\": \"[0.7928, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.3919]\", \"lift_point\": \"[0.0448, 0.3919]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of norway?\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1376, 0.2779]\", \"lift_point\": \"[0.1376, 0.2779]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80107_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7928, 0.687]\", \"lift_point\": \"[0.7928, 0.687]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0448, 0.3919]\", \"lift_point\": \"[0.0448, 0.3919]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of norway?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1376, 0.2779]\", \"lift_point\": \"[0.1376, 0.2779]\", \"typed_text\": \"\" Goal: What is the capital of Norway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80107_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80227_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.6768]\", \"lift_point\": \"[0.789, 0.6768]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80227_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.6768]\", \"lift_point\": \"[0.789, 0.6768]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.9354]\", \"lift_point\": \"[0.0697, 0.9354]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80227_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.6768]\", \"lift_point\": \"[0.789, 0.6768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.9354]\", \"lift_point\": \"[0.0697, 0.9354]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9185]\", \"lift_point\": \"[0.0711, 0.9185]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80227_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.6768]\", \"lift_point\": \"[0.789, 0.6768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.9354]\", \"lift_point\": \"[0.0697, 0.9354]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9185]\", \"lift_point\": \"[0.0711, 0.9185]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.4771]\", \"lift_point\": \"[0.1847, 0.4771]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80227_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.789, 0.6768]\", \"lift_point\": \"[0.789, 0.6768]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0697, 0.9354]\", \"lift_point\": \"[0.0697, 0.9354]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0711, 0.9185]\", \"lift_point\": \"[0.0711, 0.9185]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1847, 0.4771]\", \"lift_point\": \"[0.1847, 0.4771]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80227_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7811, 0.6981]\", \"lift_point\": \"[0.7811, 0.6981]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80404_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7811, 0.6981]\", \"lift_point\": \"[0.7811, 0.6981]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.5839]\", \"lift_point\": \"[0.0556, 0.5839]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80404_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7811, 0.6981]\", \"lift_point\": \"[0.7811, 0.6981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.5839]\", \"lift_point\": \"[0.0556, 0.5839]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80404_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7811, 0.6981]\", \"lift_point\": \"[0.7811, 0.6981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.5839]\", \"lift_point\": \"[0.0556, 0.5839]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.119, 0.4978]\", \"lift_point\": \"[0.119, 0.4978]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80404_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7811, 0.6981]\", \"lift_point\": \"[0.7811, 0.6981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.5839]\", \"lift_point\": \"[0.0556, 0.5839]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.119, 0.4978]\", \"lift_point\": \"[0.119, 0.4978]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2578, 0.1609]\", \"lift_point\": \"[0.2578, 0.1609]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80404_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7811, 0.6981]\", \"lift_point\": \"[0.7811, 0.6981]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0556, 0.5839]\", \"lift_point\": \"[0.0556, 0.5839]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rate\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.119, 0.4978]\", \"lift_point\": \"[0.119, 0.4978]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2578, 0.1609]\", \"lift_point\": \"[0.2578, 0.1609]\", \"typed_text\": \"\" Goal: What's the news about the US dollar exchange rate?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80404_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6569]\", \"lift_point\": \"[0.7875, 0.6569]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80444_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6569]\", \"lift_point\": \"[0.7875, 0.6569]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.3991]\", \"lift_point\": \"[0.0662, 0.3991]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80444_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6569]\", \"lift_point\": \"[0.7875, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.3991]\", \"lift_point\": \"[0.0662, 0.3991]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80444_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6569]\", \"lift_point\": \"[0.7875, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.3991]\", \"lift_point\": \"[0.0662, 0.3991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4434]\", \"lift_point\": \"[0.1319, 0.4434]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80444_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6569]\", \"lift_point\": \"[0.7875, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.3991]\", \"lift_point\": \"[0.0662, 0.3991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4434]\", \"lift_point\": \"[0.1319, 0.4434]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80444_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6569]\", \"lift_point\": \"[0.7875, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.3991]\", \"lift_point\": \"[0.0662, 0.3991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4434]\", \"lift_point\": \"[0.1319, 0.4434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.2392]\", \"lift_point\": \"[0.1075, 0.2392]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80444_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7875, 0.6569]\", \"lift_point\": \"[0.7875, 0.6569]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0662, 0.3991]\", \"lift_point\": \"[0.0662, 0.3991]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"malaysia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4434]\", \"lift_point\": \"[0.1319, 0.4434]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1075, 0.2392]\", \"lift_point\": \"[0.1075, 0.2392]\", \"typed_text\": \"\" Goal: What's the news in Malaysia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80444_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [TYPE,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.887, 0.4448]\", \"lift_point\": \"[0.887, 0.4448]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Denver\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.113, 0.6793]\", \"lift_point\": \"[0.113, 0.6793]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.6655]\", \"lift_point\": \"[0.1318, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1347, 0.6655]\", \"lift_point\": \"[0.1347, 0.6655]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.073, 0.9065]\", \"lift_point\": \"[0.073, 0.9065]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5976, 0.4858]\", \"lift_point\": \"[0.5976, 0.4858]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.124, 0.6216]\", \"lift_point\": \"[0.124, 0.6216]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5976, 0.4858]\", \"lift_point\": \"[0.5976, 0.4858]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5945, 0.4888]\", \"lift_point\": \"[0.5945, 0.4888]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0756, 0.7041]\", \"lift_point\": \"[0.0756, 0.7041]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5976, 0.4858]\", \"lift_point\": \"[0.5976, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5945, 0.4888]\", \"lift_point\": \"[0.5945, 0.4888]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5902, 0.4888]\", \"lift_point\": \"[0.5902, 0.4888]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.143, 0.5815]\", \"lift_point\": \"[0.143, 0.5815]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.9022]\", \"lift_point\": \"[0.0645, 0.9022]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" good restaurant in Denver?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5976, 0.4858]\", \"lift_point\": \"[0.5976, 0.4858]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5945, 0.4888]\", \"lift_point\": \"[0.5945, 0.4888]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5902, 0.4888]\", \"lift_point\": \"[0.5902, 0.4888]\", \"typed_text\": \"\" Goal: What's a good restaurant in Denver?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80515_20.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.883, 0.3744]\", \"lift_point\": \"[0.883, 0.3744]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.883, 0.3744]\", \"lift_point\": \"[0.883, 0.3744]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Samsung TV cost\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.883, 0.3744]\", \"lift_point\": \"[0.883, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Samsung TV cost\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.883, 0.3744]\", \"lift_point\": \"[0.883, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Samsung TV cost\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.883, 0.3744]\", \"lift_point\": \"[0.883, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Samsung TV cost\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7759, 0.2591]\", \"lift_point\": \"[0.7759, 0.2591]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.883, 0.3744]\", \"lift_point\": \"[0.883, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Samsung TV cost\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7759, 0.2591]\", \"lift_point\": \"[0.7759, 0.2591]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7759, 0.2591]\", \"lift_point\": \"[0.7759, 0.2591]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8984, 0.322]\", \"lift_point\": \"[0.8984, 0.322]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.883, 0.3744]\", \"lift_point\": \"[0.883, 0.3744]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Samsung TV cost\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7759, 0.2591]\", \"lift_point\": \"[0.7759, 0.2591]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7759, 0.2591]\", \"lift_point\": \"[0.7759, 0.2591]\", \"typed_text\": \"\" Goal: How much does the Samsung TV cost?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80575_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.3268]\", \"lift_point\": \"[0.1128, 0.3268]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.3268]\", \"lift_point\": \"[0.1128, 0.3268]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5942, 0.5938]\", \"lift_point\": \"[0.5942, 0.5938]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.3268]\", \"lift_point\": \"[0.1128, 0.3268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5942, 0.5938]\", \"lift_point\": \"[0.5942, 0.5938]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.2918]\", \"lift_point\": \"[0.1021, 0.2918]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7888, 0.1326]\", \"lift_point\": \"[0.7888, 0.1326]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.3268]\", \"lift_point\": \"[0.1128, 0.3268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5942, 0.5938]\", \"lift_point\": \"[0.5942, 0.5938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.2918]\", \"lift_point\": \"[0.1021, 0.2918]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.7209]\", \"lift_point\": \"[0.1974, 0.7209]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.3268]\", \"lift_point\": \"[0.1128, 0.3268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5942, 0.5938]\", \"lift_point\": \"[0.5942, 0.5938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.2918]\", \"lift_point\": \"[0.1021, 0.2918]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.7209]\", \"lift_point\": \"[0.1974, 0.7209]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.245, 0.9168]\", \"lift_point\": \"[0.245, 0.9168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0569, 0.8428]\", \"lift_point\": \"[0.0569, 0.8428]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"ign\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1128, 0.3268]\", \"lift_point\": \"[0.1128, 0.3268]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5942, 0.5938]\", \"lift_point\": \"[0.5942, 0.5938]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1021, 0.2918]\", \"lift_point\": \"[0.1021, 0.2918]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1974, 0.7209]\", \"lift_point\": \"[0.1974, 0.7209]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.245, 0.9168]\", \"lift_point\": \"[0.245, 0.9168]\", \"typed_text\": \"\" Goal: What's the latest video from IGN?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80634_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.6261]\", \"lift_point\": \"[0.0614, 0.6261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.6261]\", \"lift_point\": \"[0.0614, 0.6261]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Austin\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.6261]\", \"lift_point\": \"[0.0614, 0.6261]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Austin\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n Austin\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.6261]\", \"lift_point\": \"[0.0614, 0.6261]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Austin\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n Austin\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0995, 0.7679]\", \"lift_point\": \"[0.0995, 0.7679]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.6261]\", \"lift_point\": \"[0.0614, 0.6261]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Austin\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n Austin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0995, 0.7679]\", \"lift_point\": \"[0.0995, 0.7679]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.6261]\", \"lift_point\": \"[0.0614, 0.6261]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Austin\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n Austin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0995, 0.7679]\", \"lift_point\": \"[0.0995, 0.7679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8225, 0.4878]\", \"lift_point\": \"[0.8225, 0.4878]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8143, 0.6769]\", \"lift_point\": \"[0.8143, 0.6769]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7906, 0.696]\", \"lift_point\": \"[0.7906, 0.696]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.6261]\", \"lift_point\": \"[0.0614, 0.6261]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"2 bedroom apartment rent for in Austin\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"n Austin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0995, 0.7679]\", \"lift_point\": \"[0.0995, 0.7679]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8225, 0.4878]\", \"lift_point\": \"[0.8225, 0.4878]\", \"typed_text\": \"\" Goal: How much does a 2 bedroom apartment rent for in Austin?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80653_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80753_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.7229]\", \"lift_point\": \"[0.783, 0.7229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80753_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.7229]\", \"lift_point\": \"[0.783, 0.7229]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.954]\", \"lift_point\": \"[0.0479, 0.954]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80753_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.7229]\", \"lift_point\": \"[0.783, 0.7229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.954]\", \"lift_point\": \"[0.0479, 0.954]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2316, 0.7713]\", \"lift_point\": \"[0.2316, 0.7713]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80753_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.783, 0.7229]\", \"lift_point\": \"[0.783, 0.7229]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0479, 0.954]\", \"lift_point\": \"[0.0479, 0.954]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2316, 0.7713]\", \"lift_point\": \"[0.2316, 0.7713]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito window>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80753_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7801, 0.6869]\", \"lift_point\": \"[0.7801, 0.6869]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80943_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7801, 0.6869]\", \"lift_point\": \"[0.7801, 0.6869]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.6229]\", \"lift_point\": \"[0.0724, 0.6229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80943_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7801, 0.6869]\", \"lift_point\": \"[0.7801, 0.6869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.6229]\", \"lift_point\": \"[0.0724, 0.6229]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotels in Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80943_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7801, 0.6869]\", \"lift_point\": \"[0.7801, 0.6869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.6229]\", \"lift_point\": \"[0.0724, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotels in Mexico city\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1488, 0.4352]\", \"lift_point\": \"[0.1488, 0.4352]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80943_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7801, 0.6869]\", \"lift_point\": \"[0.7801, 0.6869]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0724, 0.6229]\", \"lift_point\": \"[0.0724, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Hotels in Mexico city\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1488, 0.4352]\", \"lift_point\": \"[0.1488, 0.4352]\", \"typed_text\": \"\" Goal: Search for hotels in Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80943_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7848, 0.6821]\", \"lift_point\": \"[0.7848, 0.6821]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80950_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7848, 0.6821]\", \"lift_point\": \"[0.7848, 0.6821]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80950_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7848, 0.6821]\", \"lift_point\": \"[0.7848, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80950_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7848, 0.6821]\", \"lift_point\": \"[0.7848, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0738, 0.6647]\", \"lift_point\": \"[0.0738, 0.6647]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80950_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7848, 0.6821]\", \"lift_point\": \"[0.7848, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0738, 0.6647]\", \"lift_point\": \"[0.0738, 0.6647]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Phoenix\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80950_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7848, 0.6821]\", \"lift_point\": \"[0.7848, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0738, 0.6647]\", \"lift_point\": \"[0.0738, 0.6647]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Phoenix\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1385, 0.5288]\", \"lift_point\": \"[0.1385, 0.5288]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80950_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7848, 0.6821]\", \"lift_point\": \"[0.7848, 0.6821]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7815, 0.6881]\", \"lift_point\": \"[0.7815, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0738, 0.6647]\", \"lift_point\": \"[0.0738, 0.6647]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"good restaurant in Phoenix\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1385, 0.5288]\", \"lift_point\": \"[0.1385, 0.5288]\", \"typed_text\": \"\" Goal: What's a good restaurant in Phoenix?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80950_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7869, 0.6923]\", \"lift_point\": \"[0.7869, 0.6923]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80982_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7869, 0.6923]\", \"lift_point\": \"[0.7869, 0.6923]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6892]\", \"lift_point\": \"[0.7794, 0.6892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80982_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7869, 0.6923]\", \"lift_point\": \"[0.7869, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6892]\", \"lift_point\": \"[0.7794, 0.6892]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.4615]\", \"lift_point\": \"[0.0665, 0.4615]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80982_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7869, 0.6923]\", \"lift_point\": \"[0.7869, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6892]\", \"lift_point\": \"[0.7794, 0.6892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.4615]\", \"lift_point\": \"[0.0665, 0.4615]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronics store thats open tomorrow\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80982_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7869, 0.6923]\", \"lift_point\": \"[0.7869, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6892]\", \"lift_point\": \"[0.7794, 0.6892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.4615]\", \"lift_point\": \"[0.0665, 0.4615]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronics store thats open tomorrow\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.4764]\", \"lift_point\": \"[0.1424, 0.4764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80982_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7869, 0.6923]\", \"lift_point\": \"[0.7869, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6892]\", \"lift_point\": \"[0.7794, 0.6892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.4615]\", \"lift_point\": \"[0.0665, 0.4615]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronics store thats open tomorrow\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.4764]\", \"lift_point\": \"[0.1424, 0.4764]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5759, 0.8689]\", \"lift_point\": \"[0.5759, 0.8689]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80982_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7869, 0.6923]\", \"lift_point\": \"[0.7869, 0.6923]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7794, 0.6892]\", \"lift_point\": \"[0.7794, 0.6892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0665, 0.4615]\", \"lift_point\": \"[0.0665, 0.4615]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest electronics store thats open tomorrow\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1424, 0.4764]\", \"lift_point\": \"[0.1424, 0.4764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5759, 0.8689]\", \"lift_point\": \"[0.5759, 0.8689]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open tomorrow>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/80982_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.6881]\", \"lift_point\": \"[0.7804, 0.6881]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81092_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.6881]\", \"lift_point\": \"[0.7804, 0.6881]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.4956]\", \"lift_point\": \"[0.0545, 0.4956]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81092_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.6881]\", \"lift_point\": \"[0.7804, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.4956]\", \"lift_point\": \"[0.0545, 0.4956]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81092_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.6881]\", \"lift_point\": \"[0.7804, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.4956]\", \"lift_point\": \"[0.0545, 0.4956]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1295, 0.5517]\", \"lift_point\": \"[0.1295, 0.5517]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81092_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7804, 0.6881]\", \"lift_point\": \"[0.7804, 0.6881]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0545, 0.4956]\", \"lift_point\": \"[0.0545, 0.4956]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Los Angeles?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1295, 0.5517]\", \"lift_point\": \"[0.1295, 0.5517]\", \"typed_text\": \"\" Goal: What's the weather like in Los Angeles?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81092_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.686]\", \"lift_point\": \"[0.778, 0.686]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81144_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.686]\", \"lift_point\": \"[0.778, 0.686]\", \"typed_text\": \"\" Goal: How big is a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0842, 0.5075]\", \"lift_point\": \"[0.0842, 0.5075]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81144_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.686]\", \"lift_point\": \"[0.778, 0.686]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0842, 0.5075]\", \"lift_point\": \"[0.0842, 0.5075]\", \"typed_text\": \"\" Goal: How big is a tiger?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a tiger?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81144_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.686]\", \"lift_point\": \"[0.778, 0.686]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0842, 0.5075]\", \"lift_point\": \"[0.0842, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a tiger?\" Goal: How big is a tiger?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1356, 0.3976]\", \"lift_point\": \"[0.1356, 0.3976]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81144_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.778, 0.686]\", \"lift_point\": \"[0.778, 0.686]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0842, 0.5075]\", \"lift_point\": \"[0.0842, 0.5075]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a tiger?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1356, 0.3976]\", \"lift_point\": \"[0.1356, 0.3976]\", \"typed_text\": \"\" Goal: How big is a tiger?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81144_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7738, 0.8722]\", \"lift_point\": \"[0.7738, 0.8722]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7738, 0.8722]\", \"lift_point\": \"[0.7738, 0.8722]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.8748]\", \"lift_point\": \"[0.7757, 0.8748]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7738, 0.8722]\", \"lift_point\": \"[0.7738, 0.8722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.8748]\", \"lift_point\": \"[0.7757, 0.8748]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.7603]\", \"lift_point\": \"[0.0614, 0.7603]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7738, 0.8722]\", \"lift_point\": \"[0.7738, 0.8722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.8748]\", \"lift_point\": \"[0.7757, 0.8748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.7603]\", \"lift_point\": \"[0.0614, 0.7603]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.5219]\", \"lift_point\": \"[0.0692, 0.5219]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7738, 0.8722]\", \"lift_point\": \"[0.7738, 0.8722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.8748]\", \"lift_point\": \"[0.7757, 0.8748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.7603]\", \"lift_point\": \"[0.0614, 0.7603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.5219]\", \"lift_point\": \"[0.0692, 0.5219]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Mexican restaurants\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7738, 0.8722]\", \"lift_point\": \"[0.7738, 0.8722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.8748]\", \"lift_point\": \"[0.7757, 0.8748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.7603]\", \"lift_point\": \"[0.0614, 0.7603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.5219]\", \"lift_point\": \"[0.0692, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Mexican restaurants\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1407, 0.5556]\", \"lift_point\": \"[0.1407, 0.5556]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7738, 0.8722]\", \"lift_point\": \"[0.7738, 0.8722]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7757, 0.8748]\", \"lift_point\": \"[0.7757, 0.8748]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0614, 0.7603]\", \"lift_point\": \"[0.0614, 0.7603]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0692, 0.5219]\", \"lift_point\": \"[0.0692, 0.5219]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Mexican restaurants\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1407, 0.5556]\", \"lift_point\": \"[0.1407, 0.5556]\", \"typed_text\": \"\" Goal: Search for the best Mexican restaurants>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81174_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81325_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.6764]\", \"lift_point\": \"[0.7798, 0.6764]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81325_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.6764]\", \"lift_point\": \"[0.7798, 0.6764]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.6045]\", \"lift_point\": \"[0.0727, 0.6045]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81325_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.6764]\", \"lift_point\": \"[0.7798, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.6045]\", \"lift_point\": \"[0.0727, 0.6045]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rateagainst the Swiss France\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81325_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.6764]\", \"lift_point\": \"[0.7798, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.6045]\", \"lift_point\": \"[0.0727, 0.6045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rateagainst the Swiss France\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.2672]\", \"lift_point\": \"[0.1318, 0.2672]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81325_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7798, 0.6764]\", \"lift_point\": \"[0.7798, 0.6764]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0727, 0.6045]\", \"lift_point\": \"[0.0727, 0.6045]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"US dollar exchange rateagainst the Swiss France\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1318, 0.2672]\", \"lift_point\": \"[0.1318, 0.2672]\", \"typed_text\": \"\" Goal: What's the US dollar exchange rateagainst the Swiss Franc?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81325_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Japan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.3527]\", \"lift_point\": \"[0.8924, 0.3527]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81599_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.3527]\", \"lift_point\": \"[0.8924, 0.3527]\", \"typed_text\": \"\" Goal: What's the news in Japan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in japan\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81599_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.3527]\", \"lift_point\": \"[0.8924, 0.3527]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in japan\" Goal: What's the news in Japan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1425, 0.4328]\", \"lift_point\": \"[0.1425, 0.4328]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81599_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.3527]\", \"lift_point\": \"[0.8924, 0.3527]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in japan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1425, 0.4328]\", \"lift_point\": \"[0.1425, 0.4328]\", \"typed_text\": \"\" Goal: What's the news in Japan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5775, 0.2184]\", \"lift_point\": \"[0.5775, 0.2184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81599_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8924, 0.3527]\", \"lift_point\": \"[0.8924, 0.3527]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"news in japan\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1425, 0.4328]\", \"lift_point\": \"[0.1425, 0.4328]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5775, 0.2184]\", \"lift_point\": \"[0.5775, 0.2184]\", \"typed_text\": \"\" Goal: What's the news in Japan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81599_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7789, 0.6894]\", \"lift_point\": \"[0.7789, 0.6894]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0401, 0.4589]\", \"lift_point\": \"[0.0401, 0.4589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"nearest Target?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0758, 0.5215]\", \"lift_point\": \"[0.0758, 0.5215]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1215, 0.202]\", \"lift_point\": \"[0.1215, 0.202]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2658, 0.0849]\", \"lift_point\": \"[0.2658, 0.0849]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0758, 0.5215]\", \"lift_point\": \"[0.0758, 0.5215]\", \"typed_text\": \"\" Goal: How do I get to the nearest Target?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81677_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81853_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.6197]\", \"lift_point\": \"[0.8229, 0.6197]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81853_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.6197]\", \"lift_point\": \"[0.8229, 0.6197]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.7522]\", \"lift_point\": \"[0.1985, 0.7522]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81853_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.6197]\", \"lift_point\": \"[0.8229, 0.6197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.7522]\", \"lift_point\": \"[0.1985, 0.7522]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.9301]\", \"lift_point\": \"[0.1142, 0.9301]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81853_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.6197]\", \"lift_point\": \"[0.8229, 0.6197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.7522]\", \"lift_point\": \"[0.1985, 0.7522]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.9301]\", \"lift_point\": \"[0.1142, 0.9301]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest JCPenney?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81853_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.6197]\", \"lift_point\": \"[0.8229, 0.6197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.7522]\", \"lift_point\": \"[0.1985, 0.7522]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.9301]\", \"lift_point\": \"[0.1142, 0.9301]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest JCPenney?\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3592, 0.313]\", \"lift_point\": \"[0.3592, 0.313]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81853_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8229, 0.6197]\", \"lift_point\": \"[0.8229, 0.6197]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1985, 0.7522]\", \"lift_point\": \"[0.1985, 0.7522]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1142, 0.9301]\", \"lift_point\": \"[0.1142, 0.9301]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How do I get to the nearest JCPenney?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3592, 0.313]\", \"lift_point\": \"[0.3592, 0.313]\", \"typed_text\": \"\" Goal: How do I get to the nearest JCPenney?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81853_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6864]\", \"lift_point\": \"[0.8079, 0.6864]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81887_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6864]\", \"lift_point\": \"[0.8079, 0.6864]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81887_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6864]\", \"lift_point\": \"[0.8079, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3123, 0.3374]\", \"lift_point\": \"[0.3123, 0.3374]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81887_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6864]\", \"lift_point\": \"[0.8079, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3123, 0.3374]\", \"lift_point\": \"[0.3123, 0.3374]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Papa Murphys?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81887_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6864]\", \"lift_point\": \"[0.8079, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3123, 0.3374]\", \"lift_point\": \"[0.3123, 0.3374]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Papa Murphys?\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81887_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6864]\", \"lift_point\": \"[0.8079, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3123, 0.3374]\", \"lift_point\": \"[0.3123, 0.3374]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Papa Murphys?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4011]\", \"lift_point\": \"[0.1319, 0.4011]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81887_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8079, 0.6864]\", \"lift_point\": \"[0.8079, 0.6864]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3123, 0.3374]\", \"lift_point\": \"[0.3123, 0.3374]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"menu at Papa Murphys?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1319, 0.4011]\", \"lift_point\": \"[0.1319, 0.4011]\", \"typed_text\": \"\" Goal: What's on the menu at Papa Murphy's?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81887_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9322, 0.5586]\", \"lift_point\": \"[0.9322, 0.5586]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9322, 0.5586]\", \"lift_point\": \"[0.9322, 0.5586]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Taiwan?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9322, 0.5586]\", \"lift_point\": \"[0.9322, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Taiwan?\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"acgfn bvc\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9322, 0.5586]\", \"lift_point\": \"[0.9322, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Taiwan?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"acgfn bvc\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1162, 0.2948]\", \"lift_point\": \"[0.1162, 0.2948]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9322, 0.5586]\", \"lift_point\": \"[0.9322, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Taiwan?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"acgfn bvc\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1162, 0.2948]\", \"lift_point\": \"[0.1162, 0.2948]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1056, 0.2084]\", \"lift_point\": \"[0.1056, 0.2084]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9322, 0.5586]\", \"lift_point\": \"[0.9322, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Taiwan?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"acgfn bvc\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1162, 0.2948]\", \"lift_point\": \"[0.1162, 0.2948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1056, 0.2084]\", \"lift_point\": \"[0.1056, 0.2084]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1056, 0.2133]\", \"lift_point\": \"[0.1056, 0.2133]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9322, 0.5586]\", \"lift_point\": \"[0.9322, 0.5586]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Taiwan?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"acgfn bvc\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1162, 0.2948]\", \"lift_point\": \"[0.1162, 0.2948]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1056, 0.2084]\", \"lift_point\": \"[0.1056, 0.2084]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1056, 0.2133]\", \"lift_point\": \"[0.1056, 0.2133]\", \"typed_text\": \"\" Goal: What's the news in Taiwan?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81919_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the time in New York?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81934_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9238, 0.5157]\", \"lift_point\": \"[0.9238, 0.5157]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81934_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9238, 0.5157]\", \"lift_point\": \"[0.9238, 0.5157]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81934_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9238, 0.5157]\", \"lift_point\": \"[0.9238, 0.5157]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0991, 0.4122]\", \"lift_point\": \"[0.0991, 0.4122]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81934_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9238, 0.5157]\", \"lift_point\": \"[0.9238, 0.5157]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the time in New York?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0991, 0.4122]\", \"lift_point\": \"[0.0991, 0.4122]\", \"typed_text\": \"\" Goal: What's the time in New York?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81934_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81937_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0922, 0.2703]\", \"lift_point\": \"[0.0922, 0.2703]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81937_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Brazil\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0922, 0.2703]\", \"lift_point\": \"[0.0922, 0.2703]\", \"typed_text\": \"\" Goal: What's the news in Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81937_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81953_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81953_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6591, 0.3734]\", \"lift_point\": \"[0.6591, 0.3734]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81953_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6591, 0.3734]\", \"lift_point\": \"[0.6591, 0.3734]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.366]\", \"lift_point\": \"[0.1724, 0.366]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81953_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6591, 0.3734]\", \"lift_point\": \"[0.6591, 0.3734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.366]\", \"lift_point\": \"[0.1724, 0.366]\", \"typed_text\": \"\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Trailers?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81953_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6591, 0.3734]\", \"lift_point\": \"[0.6591, 0.3734]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.366]\", \"lift_point\": \"[0.1724, 0.366]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest video from GameSpot Trailers?\" Goal: What's the latest video from GameSpot Trailers?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81953_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Install the Uber app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.2398]\", \"lift_point\": \"[0.1099, 0.2398]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.2398]\", \"lift_point\": \"[0.1099, 0.2398]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3707, 0.3448]\", \"lift_point\": \"[0.3707, 0.3448]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.2398]\", \"lift_point\": \"[0.1099, 0.2398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3707, 0.3448]\", \"lift_point\": \"[0.3707, 0.3448]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.4917]\", \"lift_point\": \"[0.28, 0.4917]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8123, 0.5924]\", \"lift_point\": \"[0.8123, 0.5924]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.2398]\", \"lift_point\": \"[0.1099, 0.2398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3707, 0.3448]\", \"lift_point\": \"[0.3707, 0.3448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.4917]\", \"lift_point\": \"[0.28, 0.4917]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9508, 0.5121]\", \"lift_point\": \"[0.9508, 0.5121]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0531, 0.0567]\", \"lift_point\": \"[0.0531, 0.0567]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.2398]\", \"lift_point\": \"[0.1099, 0.2398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3707, 0.3448]\", \"lift_point\": \"[0.3707, 0.3448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.4917]\", \"lift_point\": \"[0.28, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9508, 0.5121]\", \"lift_point\": \"[0.9508, 0.5121]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9557, 0.2983]\", \"lift_point\": \"[0.9557, 0.2983]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0474, 0.0472]\", \"lift_point\": \"[0.0474, 0.0472]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.2398]\", \"lift_point\": \"[0.1099, 0.2398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3707, 0.3448]\", \"lift_point\": \"[0.3707, 0.3448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.4917]\", \"lift_point\": \"[0.28, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9508, 0.5121]\", \"lift_point\": \"[0.9508, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9557, 0.2983]\", \"lift_point\": \"[0.9557, 0.2983]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1926, 0.7261]\", \"lift_point\": \"[0.1926, 0.7261]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.055, 0.306]\", \"lift_point\": \"[0.055, 0.306]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" Uber app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1099, 0.2398]\", \"lift_point\": \"[0.1099, 0.2398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3707, 0.3448]\", \"lift_point\": \"[0.3707, 0.3448]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.28, 0.4917]\", \"lift_point\": \"[0.28, 0.4917]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9508, 0.5121]\", \"lift_point\": \"[0.9508, 0.5121]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9557, 0.2983]\", \"lift_point\": \"[0.9557, 0.2983]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1926, 0.7261]\", \"lift_point\": \"[0.1926, 0.7261]\", \"typed_text\": \"\" Goal: Install the Uber app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/81987_14.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.61]\", \"lift_point\": \"[0.658, 0.61]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.658, 0.6093]\", \"lift_point\": \"[0.658, 0.6093]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1885, 0.3943]\", \"lift_point\": \"[0.1885, 0.3943]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6586, 0.5046]\", \"lift_point\": \"[0.6586, 0.5046]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Washington DC\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1171, 0.3396]\", \"lift_point\": \"[0.1171, 0.3396]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6586, 0.5046]\", \"lift_point\": \"[0.6586, 0.5046]\", \"typed_text\": \"\" Goal: Search for hotels in Washington DC>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82054_13.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82112_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.6362]\", \"lift_point\": \"[0.6505, 0.6362]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82112_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.6362]\", \"lift_point\": \"[0.6505, 0.6362]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.5122]\", \"lift_point\": \"[0.0673, 0.5122]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82112_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.6362]\", \"lift_point\": \"[0.6505, 0.6362]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.5122]\", \"lift_point\": \"[0.0673, 0.5122]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9139]\", \"lift_point\": \"[0.0551, 0.9139]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82112_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.6362]\", \"lift_point\": \"[0.6505, 0.6362]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.5122]\", \"lift_point\": \"[0.0673, 0.5122]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9139]\", \"lift_point\": \"[0.0551, 0.9139]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82112_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.6362]\", \"lift_point\": \"[0.6505, 0.6362]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.5122]\", \"lift_point\": \"[0.0673, 0.5122]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9139]\", \"lift_point\": \"[0.0551, 0.9139]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.4539]\", \"lift_point\": \"[0.1137, 0.4539]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82112_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6505, 0.6362]\", \"lift_point\": \"[0.6505, 0.6362]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.5122]\", \"lift_point\": \"[0.0673, 0.5122]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0551, 0.9139]\", \"lift_point\": \"[0.0551, 0.9139]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a dinosaur?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.4539]\", \"lift_point\": \"[0.1137, 0.4539]\", \"typed_text\": \"\" Goal: How big is a dinosaur?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82112_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9443, 0.4465]\", \"lift_point\": \"[0.9443, 0.4465]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82121_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9443, 0.4465]\", \"lift_point\": \"[0.9443, 0.4465]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82121_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9443, 0.4465]\", \"lift_point\": \"[0.9443, 0.4465]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1139, 0.2819]\", \"lift_point\": \"[0.1139, 0.2819]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82121_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9443, 0.4465]\", \"lift_point\": \"[0.9443, 0.4465]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the TCL TV?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1139, 0.2819]\", \"lift_point\": \"[0.1139, 0.2819]\", \"typed_text\": \"\" Goal: What's the price of the TCL TV?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82121_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8844, 0.2062]\", \"lift_point\": \"[0.8844, 0.2062]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82269_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8844, 0.2062]\", \"lift_point\": \"[0.8844, 0.2062]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest macys\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82269_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8844, 0.2062]\", \"lift_point\": \"[0.8844, 0.2062]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest macys\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.172, 0.2863]\", \"lift_point\": \"[0.172, 0.2863]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82269_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8844, 0.2062]\", \"lift_point\": \"[0.8844, 0.2062]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"how do i get to the nearest macys\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.172, 0.2863]\", \"lift_point\": \"[0.172, 0.2863]\", \"typed_text\": \"\" Goal: How do I get to the nearest Macy's?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82269_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82379_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9059, 0.3621]\", \"lift_point\": \"[0.9059, 0.3621]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82379_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9059, 0.3621]\", \"lift_point\": \"[0.9059, 0.3621]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Seoul?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82379_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9059, 0.3621]\", \"lift_point\": \"[0.9059, 0.3621]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Seoul?\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82379_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9059, 0.3621]\", \"lift_point\": \"[0.9059, 0.3621]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Seoul?\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Seoul?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82379_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82389_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9026, 0.3914]\", \"lift_point\": \"[0.9026, 0.3914]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82389_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9026, 0.3914]\", \"lift_point\": \"[0.9026, 0.3914]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new foundation on Sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82389_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9026, 0.3914]\", \"lift_point\": \"[0.9026, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new foundation on Sephora\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [PRESS_ENTER,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82389_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9026, 0.3914]\", \"lift_point\": \"[0.9026, 0.3914]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new foundation on Sephora\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new foundation on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82389_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [TYPE,PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Selena Gomez video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Selena Gomez video\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [PRESS_ENTER,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3868, 0.5671]\", \"lift_point\": \"[0.3868, 0.5671]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1797, 0.8461]\", \"lift_point\": \"[0.1797, 0.8461]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3868, 0.5671]\", \"lift_point\": \"[0.3868, 0.5671]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2292, 0.8737]\", \"lift_point\": \"[0.2292, 0.8737]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6048, 0.3794]\", \"lift_point\": \"[0.6048, 0.3794]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0506, 0.8524]\", \"lift_point\": \"[0.0506, 0.8524]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Selena Gomez video\" \"action_type\": \"PRESS_ENTER\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3868, 0.5671]\", \"lift_point\": \"[0.3868, 0.5671]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2292, 0.8737]\", \"lift_point\": \"[0.2292, 0.8737]\", \"typed_text\": \"\" Goal: Play the new Selena Gomez video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82411_12.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6647]\", \"lift_point\": \"[0.7936, 0.6647]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6647]\", \"lift_point\": \"[0.7936, 0.6647]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.709]\", \"lift_point\": \"[0.7994, 0.709]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6647]\", \"lift_point\": \"[0.7936, 0.6647]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.709]\", \"lift_point\": \"[0.7994, 0.709]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.6651]\", \"lift_point\": \"[0.0605, 0.6651]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6647]\", \"lift_point\": \"[0.7936, 0.6647]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.709]\", \"lift_point\": \"[0.7994, 0.709]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.6651]\", \"lift_point\": \"[0.0605, 0.6651]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new nail polish\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6647]\", \"lift_point\": \"[0.7936, 0.6647]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.709]\", \"lift_point\": \"[0.7994, 0.709]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.6651]\", \"lift_point\": \"[0.0605, 0.6651]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new nail polish\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6647]\", \"lift_point\": \"[0.7936, 0.6647]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.709]\", \"lift_point\": \"[0.7994, 0.709]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.6651]\", \"lift_point\": \"[0.0605, 0.6651]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new nail polish\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1256, 0.2968]\", \"lift_point\": \"[0.1256, 0.2968]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7936, 0.6647]\", \"lift_point\": \"[0.7936, 0.6647]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7994, 0.709]\", \"lift_point\": \"[0.7994, 0.709]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0605, 0.6651]\", \"lift_point\": \"[0.0605, 0.6651]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new nail polish\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1256, 0.2968]\", \"lift_point\": \"[0.1256, 0.2968]\", \"typed_text\": \"\" Goal: Search for a new nail polish>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82525_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82556_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6752]\", \"lift_point\": \"[0.799, 0.6752]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82556_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6752]\", \"lift_point\": \"[0.799, 0.6752]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.5103]\", \"lift_point\": \"[0.0645, 0.5103]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82556_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6752]\", \"lift_point\": \"[0.799, 0.6752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.5103]\", \"lift_point\": \"[0.0645, 0.5103]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipgloss\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82556_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6752]\", \"lift_point\": \"[0.799, 0.6752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.5103]\", \"lift_point\": \"[0.0645, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipgloss\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1207, 0.2382]\", \"lift_point\": \"[0.1207, 0.2382]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82556_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.799, 0.6752]\", \"lift_point\": \"[0.799, 0.6752]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0645, 0.5103]\", \"lift_point\": \"[0.0645, 0.5103]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new lipgloss\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1207, 0.2382]\", \"lift_point\": \"[0.1207, 0.2382]\", \"typed_text\": \"\" Goal: Search for a new lipgloss>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82556_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.787, 0.6992]\", \"lift_point\": \"[0.787, 0.6992]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82628_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.787, 0.6992]\", \"lift_point\": \"[0.787, 0.6992]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9416]\", \"lift_point\": \"[0.0591, 0.9416]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82628_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.787, 0.6992]\", \"lift_point\": \"[0.787, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9416]\", \"lift_point\": \"[0.0591, 0.9416]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.6302]\", \"lift_point\": \"[0.1788, 0.6302]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82628_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.787, 0.6992]\", \"lift_point\": \"[0.787, 0.6992]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.9416]\", \"lift_point\": \"[0.0591, 0.9416]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.6302]\", \"lift_point\": \"[0.1788, 0.6302]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82628_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82682_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3637, 0.8359]\", \"lift_point\": \"[0.3637, 0.8359]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82682_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3637, 0.8359]\", \"lift_point\": \"[0.3637, 0.8359]\", \"typed_text\": \"\" Goal: Open the clock>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82682_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82737_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6517, 0.1657]\", \"lift_point\": \"[0.6517, 0.1657]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82737_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6517, 0.1657]\", \"lift_point\": \"[0.6517, 0.1657]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1957, 0.3023]\", \"lift_point\": \"[0.1957, 0.3023]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82737_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6517, 0.1657]\", \"lift_point\": \"[0.6517, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1957, 0.3023]\", \"lift_point\": \"[0.1957, 0.3023]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" a new eyeshadow on Sephora\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82737_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6517, 0.1657]\", \"lift_point\": \"[0.6517, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1957, 0.3023]\", \"lift_point\": \"[0.1957, 0.3023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" a new eyeshadow on Sephora\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1362, 0.3184]\", \"lift_point\": \"[0.1362, 0.3184]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82737_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6517, 0.1657]\", \"lift_point\": \"[0.6517, 0.1657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1957, 0.3023]\", \"lift_point\": \"[0.1957, 0.3023]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" a new eyeshadow on Sephora\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1362, 0.3184]\", \"lift_point\": \"[0.1362, 0.3184]\", \"typed_text\": \"\" Goal: Search for a new eyeshadow on Sephora>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82737_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the speed of a train?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82784_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9035, 0.4397]\", \"lift_point\": \"[0.9035, 0.4397]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82784_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9035, 0.4397]\", \"lift_point\": \"[0.9035, 0.4397]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82784_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9035, 0.4397]\", \"lift_point\": \"[0.9035, 0.4397]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.5511]\", \"lift_point\": \"[0.1125, 0.5511]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82784_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9035, 0.4397]\", \"lift_point\": \"[0.9035, 0.4397]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the speed of a train?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1125, 0.5511]\", \"lift_point\": \"[0.1125, 0.5511]\", \"typed_text\": \"\" Goal: What is the speed of a train?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82784_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3691, 0.8089]\", \"lift_point\": \"[0.3691, 0.8089]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3691, 0.8089]\", \"lift_point\": \"[0.3691, 0.8089]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9049, 0.4885]\", \"lift_point\": \"[0.9049, 0.4885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3691, 0.8089]\", \"lift_point\": \"[0.3691, 0.8089]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9049, 0.4885]\", \"lift_point\": \"[0.9049, 0.4885]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4176, 0.5168]\", \"lift_point\": \"[0.4176, 0.5168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3691, 0.8089]\", \"lift_point\": \"[0.3691, 0.8089]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9049, 0.4885]\", \"lift_point\": \"[0.9049, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4176, 0.5168]\", \"lift_point\": \"[0.4176, 0.5168]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4176, 0.5168]\", \"lift_point\": \"[0.4176, 0.5168]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3691, 0.8089]\", \"lift_point\": \"[0.3691, 0.8089]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9049, 0.4885]\", \"lift_point\": \"[0.9049, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4176, 0.5168]\", \"lift_point\": \"[0.4176, 0.5168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4176, 0.5168]\", \"lift_point\": \"[0.4176, 0.5168]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6808, 0.7381]\", \"lift_point\": \"[0.6808, 0.7381]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3691, 0.8089]\", \"lift_point\": \"[0.3691, 0.8089]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9049, 0.4885]\", \"lift_point\": \"[0.9049, 0.4885]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4176, 0.5168]\", \"lift_point\": \"[0.4176, 0.5168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4176, 0.5168]\", \"lift_point\": \"[0.4176, 0.5168]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6808, 0.7381]\", \"lift_point\": \"[0.6808, 0.7381]\", \"typed_text\": \"\" Goal: Set an alarm for 12pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82792_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9064, 0.4992]\", \"lift_point\": \"[0.9064, 0.4992]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82803_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9064, 0.4992]\", \"lift_point\": \"[0.9064, 0.4992]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the new products by Samsung?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82803_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9064, 0.4992]\", \"lift_point\": \"[0.9064, 0.4992]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the new products by Samsung?\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1251, 0.4777]\", \"lift_point\": \"[0.1251, 0.4777]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82803_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9064, 0.4992]\", \"lift_point\": \"[0.9064, 0.4992]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What are the new products by Samsung?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1251, 0.4777]\", \"lift_point\": \"[0.1251, 0.4777]\", \"typed_text\": \"\" Goal: What are the new products by Samsung?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82803_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5199]\", \"lift_point\": \"[0.8789, 0.5199]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82844_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5199]\", \"lift_point\": \"[0.8789, 0.5199]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82844_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5199]\", \"lift_point\": \"[0.8789, 0.5199]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1184, 0.5375]\", \"lift_point\": \"[0.1184, 0.5375]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82844_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8789, 0.5199]\", \"lift_point\": \"[0.8789, 0.5199]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"What is the capital of Brazil?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1184, 0.5375]\", \"lift_point\": \"[0.1184, 0.5375]\", \"typed_text\": \"\" Goal: What is the capital of Brazil?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82844_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.6132]\", \"lift_point\": \"[0.1159, 0.6132]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.6132]\", \"lift_point\": \"[0.1159, 0.6132]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.6132]\", \"lift_point\": \"[0.1159, 0.6132]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8867, 0.5388]\", \"lift_point\": \"[0.8867, 0.5388]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.6132]\", \"lift_point\": \"[0.1159, 0.6132]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4068, 0.6038]\", \"lift_point\": \"[0.4068, 0.6038]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1717, 0.5435]\", \"lift_point\": \"[0.1717, 0.5435]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1137, 0.6987]\", \"lift_point\": \"[0.1137, 0.6987]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1159, 0.6132]\", \"lift_point\": \"[0.1159, 0.6132]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0596, 0.9275]\", \"lift_point\": \"[0.0596, 0.9275]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the latest news in cosmology?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4068, 0.6038]\", \"lift_point\": \"[0.4068, 0.6038]\", \"typed_text\": \"\" Goal: What's the latest news in cosmology?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82854_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82867_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8809, 0.5071]\", \"lift_point\": \"[0.8809, 0.5071]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82867_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8809, 0.5071]\", \"lift_point\": \"[0.8809, 0.5071]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82867_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8809, 0.5071]\", \"lift_point\": \"[0.8809, 0.5071]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4274]\", \"lift_point\": \"[0.1147, 0.4274]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82867_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8809, 0.5071]\", \"lift_point\": \"[0.8809, 0.5071]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Denver?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1147, 0.4274]\", \"lift_point\": \"[0.1147, 0.4274]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Denver?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82867_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.6047]\", \"lift_point\": \"[0.0461, 0.6047]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.6047]\", \"lift_point\": \"[0.0461, 0.6047]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0836]\", \"lift_point\": \"[0.0498, 0.0836]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.6047]\", \"lift_point\": \"[0.0461, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0836]\", \"lift_point\": \"[0.0498, 0.0836]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0818]\", \"lift_point\": \"[0.0543, 0.0818]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.6047]\", \"lift_point\": \"[0.0461, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0836]\", \"lift_point\": \"[0.0498, 0.0836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0818]\", \"lift_point\": \"[0.0543, 0.0818]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.3616]\", \"lift_point\": \"[0.0676, 0.3616]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.6047]\", \"lift_point\": \"[0.0461, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0836]\", \"lift_point\": \"[0.0498, 0.0836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0818]\", \"lift_point\": \"[0.0543, 0.0818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.3616]\", \"lift_point\": \"[0.0676, 0.3616]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Orlando\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.6047]\", \"lift_point\": \"[0.0461, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0836]\", \"lift_point\": \"[0.0498, 0.0836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0818]\", \"lift_point\": \"[0.0543, 0.0818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.3616]\", \"lift_point\": \"[0.0676, 0.3616]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Orlando\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1253, 0.5745]\", \"lift_point\": \"[0.1253, 0.5745]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6429, 0.3975]\", \"lift_point\": \"[0.6429, 0.3975]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0461, 0.6047]\", \"lift_point\": \"[0.0461, 0.6047]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0498, 0.0836]\", \"lift_point\": \"[0.0498, 0.0836]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0543, 0.0818]\", \"lift_point\": \"[0.0543, 0.0818]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0676, 0.3616]\", \"lift_point\": \"[0.0676, 0.3616]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Orlando\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1253, 0.5745]\", \"lift_point\": \"[0.1253, 0.5745]\", \"typed_text\": \"\" Goal: Search for hotels in Orlando>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82909_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 1pm>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82919_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 1pm>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5776]\", \"lift_point\": \"[0.8887, 0.5776]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82919_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5776]\", \"lift_point\": \"[0.8887, 0.5776]\", \"typed_text\": \"\" Goal: Set an alarm for 1pm>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Set an alarm for 1pm\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82919_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5776]\", \"lift_point\": \"[0.8887, 0.5776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Set an alarm for 1pm\" Goal: Set an alarm for 1pm>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1098, 0.4383]\", \"lift_point\": \"[0.1098, 0.4383]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82919_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8887, 0.5776]\", \"lift_point\": \"[0.8887, 0.5776]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Set an alarm for 1pm\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1098, 0.4383]\", \"lift_point\": \"[0.1098, 0.4383]\", \"typed_text\": \"\" Goal: Set an alarm for 1pm>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/82919_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83023_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.4897]\", \"lift_point\": \"[0.8027, 0.4897]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83023_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.4897]\", \"lift_point\": \"[0.8027, 0.4897]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83023_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.4897]\", \"lift_point\": \"[0.8027, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.6246]\", \"lift_point\": \"[0.1436, 0.6246]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83023_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8027, 0.4897]\", \"lift_point\": \"[0.8027, 0.4897]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from Barcelona to Mexico city\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1436, 0.6246]\", \"lift_point\": \"[0.1436, 0.6246]\", \"typed_text\": \"\" Goal: Search for flights from Barcelona to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83023_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.8312]\", \"lift_point\": \"[0.1279, 0.8312]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.8312]\", \"lift_point\": \"[0.1279, 0.8312]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.8554]\", \"lift_point\": \"[0.6816, 0.8554]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.8312]\", \"lift_point\": \"[0.1279, 0.8312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.8554]\", \"lift_point\": \"[0.6816, 0.8554]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.4414]\", \"lift_point\": \"[0.1477, 0.4414]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.8312]\", \"lift_point\": \"[0.1279, 0.8312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.8554]\", \"lift_point\": \"[0.6816, 0.8554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.4414]\", \"lift_point\": \"[0.1477, 0.4414]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants \"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.8312]\", \"lift_point\": \"[0.1279, 0.8312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.8554]\", \"lift_point\": \"[0.6816, 0.8554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.4414]\", \"lift_point\": \"[0.1477, 0.4414]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants \" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3094, 0.5558]\", \"lift_point\": \"[0.3094, 0.5558]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.8312]\", \"lift_point\": \"[0.1279, 0.8312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.8554]\", \"lift_point\": \"[0.6816, 0.8554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.4414]\", \"lift_point\": \"[0.1477, 0.4414]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3094, 0.5558]\", \"lift_point\": \"[0.3094, 0.5558]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6698, 0.4284]\", \"lift_point\": \"[0.6698, 0.4284]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4701, 0.1795]\", \"lift_point\": \"[0.4701, 0.1795]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1279, 0.8312]\", \"lift_point\": \"[0.1279, 0.8312]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6816, 0.8554]\", \"lift_point\": \"[0.6816, 0.8554]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1477, 0.4414]\", \"lift_point\": \"[0.1477, 0.4414]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"top rated burger restaurants \" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3094, 0.5558]\", \"lift_point\": \"[0.3094, 0.5558]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6698, 0.4284]\", \"lift_point\": \"[0.6698, 0.4284]\", \"typed_text\": \"\" Goal: Search for top rated burger restaurants on Google Maps>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83074_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83183_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.2129]\", \"lift_point\": \"[0.6207, 0.2129]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83183_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.2129]\", \"lift_point\": \"[0.6207, 0.2129]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.6547]\", \"lift_point\": \"[0.1294, 0.6547]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83183_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.2129]\", \"lift_point\": \"[0.6207, 0.2129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.6547]\", \"lift_point\": \"[0.1294, 0.6547]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new Taylor Swift video\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83183_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.2129]\", \"lift_point\": \"[0.6207, 0.2129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.6547]\", \"lift_point\": \"[0.1294, 0.6547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new Taylor Swift video\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1924, 0.3366]\", \"lift_point\": \"[0.1924, 0.3366]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83183_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.2129]\", \"lift_point\": \"[0.6207, 0.2129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.6547]\", \"lift_point\": \"[0.1294, 0.6547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new Taylor Swift video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1924, 0.3366]\", \"lift_point\": \"[0.1924, 0.3366]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5283, 0.4359]\", \"lift_point\": \"[0.5283, 0.4359]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83183_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6207, 0.2129]\", \"lift_point\": \"[0.6207, 0.2129]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1294, 0.6547]\", \"lift_point\": \"[0.1294, 0.6547]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" new Taylor Swift video\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1924, 0.3366]\", \"lift_point\": \"[0.1924, 0.3366]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5283, 0.4359]\", \"lift_point\": \"[0.5283, 0.4359]\", \"typed_text\": \"\" Goal: Play the new Taylor Swift video on YouTube>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83183_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.4161]\", \"lift_point\": \"[0.7818, 0.4161]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83199_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.4161]\", \"lift_point\": \"[0.7818, 0.4161]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from NYC to Mexico city\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83199_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.4161]\", \"lift_point\": \"[0.7818, 0.4161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from NYC to Mexico city\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2617, 0.5451]\", \"lift_point\": \"[0.2617, 0.5451]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83199_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7818, 0.4161]\", \"lift_point\": \"[0.7818, 0.4161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for flights from NYC to Mexico city\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2617, 0.5451]\", \"lift_point\": \"[0.2617, 0.5451]\", \"typed_text\": \"\" Goal: Search for flights from NYC to Mexico city>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83199_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8085, 0.5081]\", \"lift_point\": \"[0.8085, 0.5081]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83234_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8085, 0.5081]\", \"lift_point\": \"[0.8085, 0.5081]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83234_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8085, 0.5081]\", \"lift_point\": \"[0.8085, 0.5081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2604, 0.512]\", \"lift_point\": \"[0.2604, 0.512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83234_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8085, 0.5081]\", \"lift_point\": \"[0.8085, 0.5081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2604, 0.512]\", \"lift_point\": \"[0.2604, 0.512]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5282, 0.459]\", \"lift_point\": \"[0.5282, 0.459]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83234_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8085, 0.5081]\", \"lift_point\": \"[0.8085, 0.5081]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on Reddit this week\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2604, 0.512]\", \"lift_point\": \"[0.2604, 0.512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5282, 0.459]\", \"lift_point\": \"[0.5282, 0.459]\", \"typed_text\": \"\" Goal: What's on Reddit this week>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83234_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Amazon Music app>55555", + "response": "Action Plan: [STATUS_TASK_IMPOSSIBLE]\n; Action Decision: \"action_type\": \"STATUS_TASK_IMPOSSIBLE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83235_1.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.5589]\", \"lift_point\": \"[0.7824, 0.5589]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83263_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.5589]\", \"lift_point\": \"[0.7824, 0.5589]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [TYPE,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83263_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7824, 0.5589]\", \"lift_point\": \"[0.7824, 0.5589]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the price of the Galaxy phone on eBay?\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83263_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.3891]\", \"lift_point\": \"[0.7913, 0.3891]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83332_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.3891]\", \"lift_point\": \"[0.7913, 0.3891]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83332_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.3891]\", \"lift_point\": \"[0.7913, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2638, 0.6026]\", \"lift_point\": \"[0.2638, 0.6026]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83332_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7913, 0.3891]\", \"lift_point\": \"[0.7913, 0.3891]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Chicago?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2638, 0.6026]\", \"lift_point\": \"[0.2638, 0.6026]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Chicago?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83332_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3347, 0.474]\", \"lift_point\": \"[0.3347, 0.474]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3347, 0.474]\", \"lift_point\": \"[0.3347, 0.474]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4896, 0.4304]\", \"lift_point\": \"[0.4896, 0.4304]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3347, 0.474]\", \"lift_point\": \"[0.3347, 0.474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4896, 0.4304]\", \"lift_point\": \"[0.4896, 0.4304]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8783, 0.5749]\", \"lift_point\": \"[0.8783, 0.5749]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3347, 0.474]\", \"lift_point\": \"[0.3347, 0.474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4896, 0.4304]\", \"lift_point\": \"[0.4896, 0.4304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8783, 0.5749]\", \"lift_point\": \"[0.8783, 0.5749]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.1574]\", \"lift_point\": \"[0.885, 0.1574]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3347, 0.474]\", \"lift_point\": \"[0.3347, 0.474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4896, 0.4304]\", \"lift_point\": \"[0.4896, 0.4304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8783, 0.5749]\", \"lift_point\": \"[0.8783, 0.5749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.1574]\", \"lift_point\": \"[0.885, 0.1574]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5326]\", \"lift_point\": \"[0.876, 0.5326]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8927, 0.4245]\", \"lift_point\": \"[0.8927, 0.4245]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in theFalkland Islands?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3347, 0.474]\", \"lift_point\": \"[0.3347, 0.474]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4896, 0.4304]\", \"lift_point\": \"[0.4896, 0.4304]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8783, 0.5749]\", \"lift_point\": \"[0.8783, 0.5749]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.885, 0.1574]\", \"lift_point\": \"[0.885, 0.1574]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.876, 0.5326]\", \"lift_point\": \"[0.876, 0.5326]\", \"typed_text\": \"\" Goal: What's the news in theFalkland Islands?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83377_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83411_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9001, 0.5193]\", \"lift_point\": \"[0.9001, 0.5193]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83411_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9001, 0.5193]\", \"lift_point\": \"[0.9001, 0.5193]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in San Francisco?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83411_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9001, 0.5193]\", \"lift_point\": \"[0.9001, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in San Francisco?\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0999, 0.5128]\", \"lift_point\": \"[0.0999, 0.5128]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83411_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9001, 0.5193]\", \"lift_point\": \"[0.9001, 0.5193]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"a good restaurant in San Francisco?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0999, 0.5128]\", \"lift_point\": \"[0.0999, 0.5128]\", \"typed_text\": \"\" Goal: What's a good restaurant in San Francisco?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83411_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83429_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9103, 0.5828]\", \"lift_point\": \"[0.9103, 0.5828]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83429_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9103, 0.5828]\", \"lift_point\": \"[0.9103, 0.5828]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83429_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9103, 0.5828]\", \"lift_point\": \"[0.9103, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1035, 0.6551]\", \"lift_point\": \"[0.1035, 0.6551]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83429_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9103, 0.5828]\", \"lift_point\": \"[0.9103, 0.5828]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the weather like in Mumbai?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1035, 0.6551]\", \"lift_point\": \"[0.1035, 0.6551]\", \"typed_text\": \"\" Goal: What's the weather like in Mumbai?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83429_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.0578]\", \"lift_point\": \"[0.0475, 0.0578]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.0578]\", \"lift_point\": \"[0.0475, 0.0578]\", \"typed_text\": \"\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.2869]\", \"lift_point\": \"[0.0475, 0.2869]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.0578]\", \"lift_point\": \"[0.0475, 0.0578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.2869]\", \"lift_point\": \"[0.0475, 0.2869]\", \"typed_text\": \"\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pandora app\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.0578]\", \"lift_point\": \"[0.0475, 0.0578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.2869]\", \"lift_point\": \"[0.0475, 0.2869]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pandora app\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2133, 0.2965]\", \"lift_point\": \"[0.2133, 0.2965]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.0578]\", \"lift_point\": \"[0.0475, 0.0578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.2869]\", \"lift_point\": \"[0.0475, 0.2869]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pandora app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2133, 0.2965]\", \"lift_point\": \"[0.2133, 0.2965]\", \"typed_text\": \"\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3728, 0.3114]\", \"lift_point\": \"[0.3728, 0.3114]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.0578]\", \"lift_point\": \"[0.0475, 0.0578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.2869]\", \"lift_point\": \"[0.0475, 0.2869]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pandora app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2133, 0.2965]\", \"lift_point\": \"[0.2133, 0.2965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3728, 0.3114]\", \"lift_point\": \"[0.3728, 0.3114]\", \"typed_text\": \"\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.272, 0.4426]\", \"lift_point\": \"[0.272, 0.4426]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8933, 0.3738]\", \"lift_point\": \"[0.8933, 0.3738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3375, 0.1329]\", \"lift_point\": \"[0.3375, 0.1329]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.0578]\", \"lift_point\": \"[0.0475, 0.0578]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0475, 0.2869]\", \"lift_point\": \"[0.0475, 0.2869]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Pandora app\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2133, 0.2965]\", \"lift_point\": \"[0.2133, 0.2965]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3728, 0.3114]\", \"lift_point\": \"[0.3728, 0.3114]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.272, 0.4426]\", \"lift_point\": \"[0.272, 0.4426]\", \"typed_text\": \"\" Goal: Check the settings for the Pandora app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83479_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at Subway?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83585_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on the menu at Subway?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.901, 0.4938]\", \"lift_point\": \"[0.901, 0.4938]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83585_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.901, 0.4938]\", \"lift_point\": \"[0.901, 0.4938]\", \"typed_text\": \"\" Goal: What's on the menu at Subway?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Subway\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83585_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.901, 0.4938]\", \"lift_point\": \"[0.901, 0.4938]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Subway\" Goal: What's on the menu at Subway?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1047, 0.2424]\", \"lift_point\": \"[0.1047, 0.2424]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83585_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.901, 0.4938]\", \"lift_point\": \"[0.901, 0.4938]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Subway\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1047, 0.2424]\", \"lift_point\": \"[0.1047, 0.2424]\", \"typed_text\": \"\" Goal: What's on the menu at Subway?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83585_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6507, 0.6143]\", \"lift_point\": \"[0.6507, 0.6143]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6507, 0.6143]\", \"lift_point\": \"[0.6507, 0.6143]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.2766]\", \"lift_point\": \"[0.1788, 0.2766]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6507, 0.6143]\", \"lift_point\": \"[0.6507, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.2766]\", \"lift_point\": \"[0.1788, 0.2766]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6507, 0.6143]\", \"lift_point\": \"[0.6507, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.2766]\", \"lift_point\": \"[0.1788, 0.2766]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6507, 0.6143]\", \"lift_point\": \"[0.6507, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.2766]\", \"lift_point\": \"[0.1788, 0.2766]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6507, 0.6143]\", \"lift_point\": \"[0.6507, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.2766]\", \"lift_point\": \"[0.1788, 0.2766]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3611, 0.362]\", \"lift_point\": \"[0.3611, 0.362]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6507, 0.6143]\", \"lift_point\": \"[0.6507, 0.6143]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1788, 0.2766]\", \"lift_point\": \"[0.1788, 0.2766]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3611, 0.362]\", \"lift_point\": \"[0.3611, 0.362]\", \"typed_text\": \"\" Goal: Open the contacts app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83616_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8222, 0.8442]\", \"lift_point\": \"[0.8222, 0.8442]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8222, 0.8442]\", \"lift_point\": \"[0.8222, 0.8442]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.4251]\", \"lift_point\": \"[0.0591, 0.4251]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8222, 0.8442]\", \"lift_point\": \"[0.8222, 0.8442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.4251]\", \"lift_point\": \"[0.0591, 0.4251]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8222, 0.8442]\", \"lift_point\": \"[0.8222, 0.8442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.4251]\", \"lift_point\": \"[0.0591, 0.4251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.2102]\", \"lift_point\": \"[0.1053, 0.2102]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8222, 0.8442]\", \"lift_point\": \"[0.8222, 0.8442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.4251]\", \"lift_point\": \"[0.0591, 0.4251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.2102]\", \"lift_point\": \"[0.1053, 0.2102]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1216, 0.8219]\", \"lift_point\": \"[0.1216, 0.8219]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8222, 0.8442]\", \"lift_point\": \"[0.8222, 0.8442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.4251]\", \"lift_point\": \"[0.0591, 0.4251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.2102]\", \"lift_point\": \"[0.1053, 0.2102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1216, 0.8219]\", \"lift_point\": \"[0.1216, 0.8219]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1216, 0.8219]\", \"lift_point\": \"[0.1216, 0.8219]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9042, 0.4074]\", \"lift_point\": \"[0.9042, 0.4074]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8222, 0.8442]\", \"lift_point\": \"[0.8222, 0.8442]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0591, 0.4251]\", \"lift_point\": \"[0.0591, 0.4251]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Spotify\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1053, 0.2102]\", \"lift_point\": \"[0.1053, 0.2102]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1216, 0.8219]\", \"lift_point\": \"[0.1216, 0.8219]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1216, 0.8219]\", \"lift_point\": \"[0.1216, 0.8219]\", \"typed_text\": \"\" Goal: Check the settings for the Spotify app>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83829_9.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new private window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5286, 0.6744]\", \"lift_point\": \"[0.5286, 0.6744]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83835_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5286, 0.6744]\", \"lift_point\": \"[0.5286, 0.6744]\", \"typed_text\": \"\" Goal: Open a new private window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83835_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5286, 0.6744]\", \"lift_point\": \"[0.5286, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Open a new private window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.8512]\", \"lift_point\": \"[0.325, 0.8512]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83835_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5286, 0.6744]\", \"lift_point\": \"[0.5286, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.8512]\", \"lift_point\": \"[0.325, 0.8512]\", \"typed_text\": \"\" Goal: Open a new private window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9321]\", \"lift_point\": \"[0.0604, 0.9321]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83835_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5286, 0.6744]\", \"lift_point\": \"[0.5286, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.8512]\", \"lift_point\": \"[0.325, 0.8512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9321]\", \"lift_point\": \"[0.0604, 0.9321]\", \"typed_text\": \"\" Goal: Open a new private window in Chrome>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1552, 0.6466]\", \"lift_point\": \"[0.1552, 0.6466]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83835_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5286, 0.6744]\", \"lift_point\": \"[0.5286, 0.6744]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.325, 0.8512]\", \"lift_point\": \"[0.325, 0.8512]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0604, 0.9321]\", \"lift_point\": \"[0.0604, 0.9321]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1552, 0.6466]\", \"lift_point\": \"[0.1552, 0.6466]\", \"typed_text\": \"\" Goal: Open a new private window in Chrome>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83835_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83865_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.1854]\", \"lift_point\": \"[0.7758, 0.1854]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83865_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.1854]\", \"lift_point\": \"[0.7758, 0.1854]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.8085]\", \"lift_point\": \"[0.0623, 0.8085]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83865_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.1854]\", \"lift_point\": \"[0.7758, 0.1854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.8085]\", \"lift_point\": \"[0.0623, 0.8085]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.4269]\", \"lift_point\": \"[0.0744, 0.4269]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83865_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.1854]\", \"lift_point\": \"[0.7758, 0.1854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.8085]\", \"lift_point\": \"[0.0623, 0.8085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.4269]\", \"lift_point\": \"[0.0744, 0.4269]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open today\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83865_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.1854]\", \"lift_point\": \"[0.7758, 0.1854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.8085]\", \"lift_point\": \"[0.0623, 0.8085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.4269]\", \"lift_point\": \"[0.0744, 0.4269]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open today\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1136, 0.3308]\", \"lift_point\": \"[0.1136, 0.3308]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83865_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7758, 0.1854]\", \"lift_point\": \"[0.7758, 0.1854]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0623, 0.8085]\", \"lift_point\": \"[0.0623, 0.8085]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0744, 0.4269]\", \"lift_point\": \"[0.0744, 0.4269]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Find the nearest electronics store thats open today\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1136, 0.3308]\", \"lift_point\": \"[0.1136, 0.3308]\", \"typed_text\": \"\" Goal: Find the nearest electronics store that's open today>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83865_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.6889]\", \"lift_point\": \"[0.7907, 0.6889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83901_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.6889]\", \"lift_point\": \"[0.7907, 0.6889]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.6889]\", \"lift_point\": \"[0.7907, 0.6889]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83901_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.6889]\", \"lift_point\": \"[0.7907, 0.6889]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7907, 0.6889]\", \"lift_point\": \"[0.7907, 0.6889]\", \"typed_text\": \"\" Goal: Open a new Chrome private tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83901_3.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83948_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.1117]\", \"lift_point\": \"[0.8909, 0.1117]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83948_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.1117]\", \"lift_point\": \"[0.8909, 0.1117]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"NEW FOUNDATION PRODUCT\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83948_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.1117]\", \"lift_point\": \"[0.8909, 0.1117]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"NEW FOUNDATION PRODUCT\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3729]\", \"lift_point\": \"[0.1315, 0.3729]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83948_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8909, 0.1117]\", \"lift_point\": \"[0.8909, 0.1117]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"NEW FOUNDATION PRODUCT\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1315, 0.3729]\", \"lift_point\": \"[0.1315, 0.3729]\", \"typed_text\": \"\" Goal: Search for a new foundation (skincare) product>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83948_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in London>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83954_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.1359]\", \"lift_point\": \"[0.8872, 0.1359]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83954_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.1359]\", \"lift_point\": \"[0.8872, 0.1359]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0723, 0.3912]\", \"lift_point\": \"[0.0723, 0.3912]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83954_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.1359]\", \"lift_point\": \"[0.8872, 0.1359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0723, 0.3912]\", \"lift_point\": \"[0.0723, 0.3912]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HOTELS IN LONDON\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83954_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.1359]\", \"lift_point\": \"[0.8872, 0.1359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0723, 0.3912]\", \"lift_point\": \"[0.0723, 0.3912]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HOTELS IN LONDON\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1452, 0.5949]\", \"lift_point\": \"[0.1452, 0.5949]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83954_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8872, 0.1359]\", \"lift_point\": \"[0.8872, 0.1359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0723, 0.3912]\", \"lift_point\": \"[0.0723, 0.3912]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"HOTELS IN LONDON\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1452, 0.5949]\", \"lift_point\": \"[0.1452, 0.5949]\", \"typed_text\": \"\" Goal: Search for hotels in London>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/83954_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.1225]\", \"lift_point\": \"[0.8814, 0.1225]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84005_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.1225]\", \"lift_point\": \"[0.8814, 0.1225]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84005_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.1225]\", \"lift_point\": \"[0.8814, 0.1225]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1409, 0.4278]\", \"lift_point\": \"[0.1409, 0.4278]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84005_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8814, 0.1225]\", \"lift_point\": \"[0.8814, 0.1225]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How much does a 3 bedroom apartment rent for in Boston?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1409, 0.4278]\", \"lift_point\": \"[0.1409, 0.4278]\", \"typed_text\": \"\" Goal: How much does a 3 bedroom apartment rent for in Boston?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84005_4.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.3892]\", \"lift_point\": \"[0.0617, 0.3892]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.3892]\", \"lift_point\": \"[0.0617, 0.3892]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9182]\", \"lift_point\": \"[0.0649, 0.9182]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.3892]\", \"lift_point\": \"[0.0617, 0.3892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9182]\", \"lift_point\": \"[0.0649, 0.9182]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the Galaxy phone on eBay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.3892]\", \"lift_point\": \"[0.0617, 0.3892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9182]\", \"lift_point\": \"[0.0649, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the Galaxy phone on eBay\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.4874]\", \"lift_point\": \"[0.1406, 0.4874]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.3892]\", \"lift_point\": \"[0.0617, 0.3892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9182]\", \"lift_point\": \"[0.0649, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the Galaxy phone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.4874]\", \"lift_point\": \"[0.1406, 0.4874]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4036, 0.418]\", \"lift_point\": \"[0.4036, 0.418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.3892]\", \"lift_point\": \"[0.0617, 0.3892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9182]\", \"lift_point\": \"[0.0649, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the Galaxy phone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.4874]\", \"lift_point\": \"[0.1406, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4036, 0.418]\", \"lift_point\": \"[0.4036, 0.418]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6794, 0.2221]\", \"lift_point\": \"[0.6794, 0.2221]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7825, 0.6728]\", \"lift_point\": \"[0.7825, 0.6728]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7946, 0.7096]\", \"lift_point\": \"[0.7946, 0.7096]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0617, 0.3892]\", \"lift_point\": \"[0.0617, 0.3892]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.9182]\", \"lift_point\": \"[0.0649, 0.9182]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" price of the Galaxy phone on eBay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1406, 0.4874]\", \"lift_point\": \"[0.1406, 0.4874]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4036, 0.418]\", \"lift_point\": \"[0.4036, 0.418]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6794, 0.2221]\", \"lift_point\": \"[0.6794, 0.2221]\", \"typed_text\": \"\" Goal: What's the price of the Galaxy phone on eBay?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84111_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\" Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6999]\", \"lift_point\": \"[0.7775, 0.6999]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6999]\", \"lift_point\": \"[0.7775, 0.6999]\", \"typed_text\": \"\" Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.075, 0.5984]\", \"lift_point\": \"[0.075, 0.5984]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6999]\", \"lift_point\": \"[0.7775, 0.6999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.075, 0.5984]\", \"lift_point\": \"[0.075, 0.5984]\", \"typed_text\": \"\" Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [TYPE,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6999]\", \"lift_point\": \"[0.7775, 0.6999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.075, 0.5984]\", \"lift_point\": \"[0.075, 0.5984]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?\" Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" cordless chainsaw\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6999]\", \"lift_point\": \"[0.7775, 0.6999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.075, 0.5984]\", \"lift_point\": \"[0.075, 0.5984]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" cordless chainsaw\" Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1247, 0.7833]\", \"lift_point\": \"[0.1247, 0.7833]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.6229]\", \"lift_point\": \"[0.902, 0.6229]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7775, 0.6999]\", \"lift_point\": \"[0.7775, 0.6999]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.075, 0.5984]\", \"lift_point\": \"[0.075, 0.5984]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \" cordless chainsaw\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1247, 0.7833]\", \"lift_point\": \"[0.1247, 0.7833]\", \"typed_text\": \"\" Goal: What's the price of the EGO 14-in 56-Volt Brushless Cordless Chainsaw?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84198_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What time is it in Berlin?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84210_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What time is it in Berlin?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7945, 0.7061]\", \"lift_point\": \"[0.7945, 0.7061]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84210_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7945, 0.7061]\", \"lift_point\": \"[0.7945, 0.7061]\", \"typed_text\": \"\" Goal: What time is it in Berlin?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.6814]\", \"lift_point\": \"[0.0722, 0.6814]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84210_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7945, 0.7061]\", \"lift_point\": \"[0.7945, 0.7061]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.6814]\", \"lift_point\": \"[0.0722, 0.6814]\", \"typed_text\": \"\" Goal: What time is it in Berlin?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in berlin\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84210_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7945, 0.7061]\", \"lift_point\": \"[0.7945, 0.7061]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.6814]\", \"lift_point\": \"[0.0722, 0.6814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in berlin\" Goal: What time is it in Berlin?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.3394]\", \"lift_point\": \"[0.1426, 0.3394]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84210_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7945, 0.7061]\", \"lift_point\": \"[0.7945, 0.7061]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0722, 0.6814]\", \"lift_point\": \"[0.0722, 0.6814]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"time in berlin\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1426, 0.3394]\", \"lift_point\": \"[0.1426, 0.3394]\", \"typed_text\": \"\" Goal: What time is it in Berlin?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84210_6.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9083]\", \"lift_point\": \"[0.0646, 0.9083]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9083]\", \"lift_point\": \"[0.0646, 0.9083]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Las Vegas?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9083]\", \"lift_point\": \"[0.0646, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Las Vegas?\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1349, 0.4538]\", \"lift_point\": \"[0.1349, 0.4538]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9083]\", \"lift_point\": \"[0.0646, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Las Vegas?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1349, 0.4538]\", \"lift_point\": \"[0.1349, 0.4538]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3392, 0.4296]\", \"lift_point\": \"[0.3392, 0.4296]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9083]\", \"lift_point\": \"[0.0646, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Las Vegas?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1349, 0.4538]\", \"lift_point\": \"[0.1349, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3392, 0.4296]\", \"lift_point\": \"[0.3392, 0.4296]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8815, 0.5826]\", \"lift_point\": \"[0.8815, 0.5826]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9083]\", \"lift_point\": \"[0.0646, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Las Vegas?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1349, 0.4538]\", \"lift_point\": \"[0.1349, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3392, 0.4296]\", \"lift_point\": \"[0.3392, 0.4296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8815, 0.5826]\", \"lift_point\": \"[0.8815, 0.5826]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.8592]\", \"lift_point\": \"[0.902, 0.8592]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4989, 0.6957]\", \"lift_point\": \"[0.4989, 0.6957]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1298, 0.4095]\", \"lift_point\": \"[0.1298, 0.4095]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0646, 0.9083]\", \"lift_point\": \"[0.0646, 0.9083]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats a good restaurant in Las Vegas?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1349, 0.4538]\", \"lift_point\": \"[0.1349, 0.4538]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3392, 0.4296]\", \"lift_point\": \"[0.3392, 0.4296]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8815, 0.5826]\", \"lift_point\": \"[0.8815, 0.5826]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.902, 0.8592]\", \"lift_point\": \"[0.902, 0.8592]\", \"typed_text\": \"\" Goal: What's a good restaurant in Las Vegas?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84292_11.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5067, 0.7115]\", \"lift_point\": \"[0.5067, 0.7115]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5067, 0.7115]\", \"lift_point\": \"[0.5067, 0.7115]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.4263]\", \"lift_point\": \"[0.1179, 0.4263]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5067, 0.7115]\", \"lift_point\": \"[0.5067, 0.7115]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.4263]\", \"lift_point\": \"[0.1179, 0.4263]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.9161]\", \"lift_point\": \"[0.0575, 0.9161]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5067, 0.7115]\", \"lift_point\": \"[0.5067, 0.7115]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.4263]\", \"lift_point\": \"[0.1179, 0.4263]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.9161]\", \"lift_point\": \"[0.0575, 0.9161]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5067, 0.7115]\", \"lift_point\": \"[0.5067, 0.7115]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.4263]\", \"lift_point\": \"[0.1179, 0.4263]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.9161]\", \"lift_point\": \"[0.0575, 0.9161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1273, 0.3493]\", \"lift_point\": \"[0.1273, 0.3493]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5067, 0.7115]\", \"lift_point\": \"[0.5067, 0.7115]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.4263]\", \"lift_point\": \"[0.1179, 0.4263]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.9161]\", \"lift_point\": \"[0.0575, 0.9161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1273, 0.3493]\", \"lift_point\": \"[0.1273, 0.3493]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3102, 0.4882]\", \"lift_point\": \"[0.3102, 0.4882]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5067, 0.7115]\", \"lift_point\": \"[0.5067, 0.7115]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1179, 0.4263]\", \"lift_point\": \"[0.1179, 0.4263]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0575, 0.9161]\", \"lift_point\": \"[0.0575, 0.9161]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Search for hotels in Buenos aires\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1273, 0.3493]\", \"lift_point\": \"[0.1273, 0.3493]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3102, 0.4882]\", \"lift_point\": \"[0.3102, 0.4882]\", \"typed_text\": \"\" Goal: Search for hotels in Buenos aires>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84387_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3823, 0.189]\", \"lift_point\": \"[0.3823, 0.189]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9136, 0.9292]\", \"lift_point\": \"[0.9136, 0.9292]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.3526]\", \"lift_point\": \"[0.066, 0.3526]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8915, 0.5425]\", \"lift_point\": \"[0.8915, 0.5425]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.3526]\", \"lift_point\": \"[0.066, 0.3526]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.3526]\", \"lift_point\": \"[0.066, 0.3526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2468, 0.1853]\", \"lift_point\": \"[0.2468, 0.1853]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.3526]\", \"lift_point\": \"[0.066, 0.3526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2468, 0.1853]\", \"lift_point\": \"[0.2468, 0.1853]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4152]\", \"lift_point\": \"[0.584, 0.4152]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2, 0.5]\", \"lift_point\": \"[0.8, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.066, 0.3526]\", \"lift_point\": \"[0.066, 0.3526]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2468, 0.1853]\", \"lift_point\": \"[0.2468, 0.1853]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.584, 0.4152]\", \"lift_point\": \"[0.584, 0.4152]\", \"typed_text\": \"\" Goal: What's on my calendar for the rest of the week?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84571_16.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84617_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6558, 0.1738]\", \"lift_point\": \"[0.6558, 0.1738]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84617_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6558, 0.1738]\", \"lift_point\": \"[0.6558, 0.1738]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1922, 0.5511]\", \"lift_point\": \"[0.1922, 0.5511]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84617_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6558, 0.1738]\", \"lift_point\": \"[0.6558, 0.1738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1922, 0.5511]\", \"lift_point\": \"[0.1922, 0.5511]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0532, 0.8873]\", \"lift_point\": \"[0.0532, 0.8873]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84617_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6558, 0.1738]\", \"lift_point\": \"[0.6558, 0.1738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1922, 0.5511]\", \"lift_point\": \"[0.1922, 0.5511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0532, 0.8873]\", \"lift_point\": \"[0.0532, 0.8873]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84617_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6558, 0.1738]\", \"lift_point\": \"[0.6558, 0.1738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1922, 0.5511]\", \"lift_point\": \"[0.1922, 0.5511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0532, 0.8873]\", \"lift_point\": \"[0.0532, 0.8873]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.3598]\", \"lift_point\": \"[0.3452, 0.3598]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84617_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6558, 0.1738]\", \"lift_point\": \"[0.6558, 0.1738]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1922, 0.5511]\", \"lift_point\": \"[0.1922, 0.5511]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0532, 0.8873]\", \"lift_point\": \"[0.0532, 0.8873]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"How big is a blue whale?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3452, 0.3598]\", \"lift_point\": \"[0.3452, 0.3598]\", \"typed_text\": \"\" Goal: How big is a blue whale?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84617_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84708_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6413, 0.1582]\", \"lift_point\": \"[0.6413, 0.1582]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84708_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6413, 0.1582]\", \"lift_point\": \"[0.6413, 0.1582]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.834]\", \"lift_point\": \"[0.1724, 0.834]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84708_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6413, 0.1582]\", \"lift_point\": \"[0.6413, 0.1582]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.834]\", \"lift_point\": \"[0.1724, 0.834]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9097]\", \"lift_point\": \"[0.0534, 0.9097]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84708_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6413, 0.1582]\", \"lift_point\": \"[0.6413, 0.1582]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.834]\", \"lift_point\": \"[0.1724, 0.834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9097]\", \"lift_point\": \"[0.0534, 0.9097]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Las Vegas\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84708_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6413, 0.1582]\", \"lift_point\": \"[0.6413, 0.1582]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.834]\", \"lift_point\": \"[0.1724, 0.834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9097]\", \"lift_point\": \"[0.0534, 0.9097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Las Vegas\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1708, 0.4418]\", \"lift_point\": \"[0.1708, 0.4418]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84708_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6413, 0.1582]\", \"lift_point\": \"[0.6413, 0.1582]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1724, 0.834]\", \"lift_point\": \"[0.1724, 0.834]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0534, 0.9097]\", \"lift_point\": \"[0.0534, 0.9097]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"hotels in Las Vegas\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1708, 0.4418]\", \"lift_point\": \"[0.1708, 0.4418]\", \"typed_text\": \"\" Goal: Search for hotels in Las Vegas>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84708_7.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in India?>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6166, 0.1636]\", \"lift_point\": \"[0.6166, 0.1636]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6166, 0.1636]\", \"lift_point\": \"[0.6166, 0.1636]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1983, 0.6547]\", \"lift_point\": \"[0.1983, 0.6547]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6166, 0.1636]\", \"lift_point\": \"[0.6166, 0.1636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1983, 0.6547]\", \"lift_point\": \"[0.1983, 0.6547]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.06, 0.9095]\", \"lift_point\": \"[0.06, 0.9095]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6166, 0.1636]\", \"lift_point\": \"[0.6166, 0.1636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1983, 0.6547]\", \"lift_point\": \"[0.1983, 0.6547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.06, 0.9095]\", \"lift_point\": \"[0.06, 0.9095]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6166, 0.1636]\", \"lift_point\": \"[0.6166, 0.1636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1983, 0.6547]\", \"lift_point\": \"[0.1983, 0.6547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.06, 0.9095]\", \"lift_point\": \"[0.06, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\" Goal: What's the news in India?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1841, 0.5421]\", \"lift_point\": \"[0.1841, 0.5421]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6166, 0.1636]\", \"lift_point\": \"[0.6166, 0.1636]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1983, 0.6547]\", \"lift_point\": \"[0.1983, 0.6547]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.06, 0.9095]\", \"lift_point\": \"[0.06, 0.9095]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in India?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1841, 0.5421]\", \"lift_point\": \"[0.1841, 0.5421]\", \"typed_text\": \"\" Goal: What's the news in India?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84738_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.86]\", \"lift_point\": \"[0.332, 0.86]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.86]\", \"lift_point\": \"[0.332, 0.86]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.518]\", \"lift_point\": \"[0.9056, 0.518]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.86]\", \"lift_point\": \"[0.332, 0.86]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.518]\", \"lift_point\": \"[0.9056, 0.518]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6016, 0.4149]\", \"lift_point\": \"[0.6016, 0.4149]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.86]\", \"lift_point\": \"[0.332, 0.86]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.518]\", \"lift_point\": \"[0.9056, 0.518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6016, 0.4149]\", \"lift_point\": \"[0.6016, 0.4149]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4208, 0.4956]\", \"lift_point\": \"[0.4208, 0.4956]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.86]\", \"lift_point\": \"[0.332, 0.86]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.518]\", \"lift_point\": \"[0.9056, 0.518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6016, 0.4149]\", \"lift_point\": \"[0.6016, 0.4149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4208, 0.4956]\", \"lift_point\": \"[0.4208, 0.4956]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6727, 0.7409]\", \"lift_point\": \"[0.6727, 0.7409]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.332, 0.86]\", \"lift_point\": \"[0.332, 0.86]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.9056, 0.518]\", \"lift_point\": \"[0.9056, 0.518]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6016, 0.4149]\", \"lift_point\": \"[0.6016, 0.4149]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4208, 0.4956]\", \"lift_point\": \"[0.4208, 0.4956]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6727, 0.7409]\", \"lift_point\": \"[0.6727, 0.7409]\", \"typed_text\": \"\" Goal: Set an alarm for 7am>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/84861_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.6684]\", \"lift_point\": \"[0.776, 0.6684]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85034_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.6684]\", \"lift_point\": \"[0.776, 0.6684]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85034_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.6684]\", \"lift_point\": \"[0.776, 0.6684]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.5359]\", \"lift_point\": \"[0.1433, 0.5359]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85034_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.6684]\", \"lift_point\": \"[0.776, 0.6684]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.5359]\", \"lift_point\": \"[0.1433, 0.5359]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2512, 0.2042]\", \"lift_point\": \"[0.2512, 0.2042]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85034_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.776, 0.6684]\", \"lift_point\": \"[0.776, 0.6684]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Myanmar?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1433, 0.5359]\", \"lift_point\": \"[0.1433, 0.5359]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2512, 0.2042]\", \"lift_point\": \"[0.2512, 0.2042]\", \"typed_text\": \"\" Goal: What's the news in Myanmar?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85034_5.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3493]\", \"lift_point\": \"[0.0649, 0.3493]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3493]\", \"lift_point\": \"[0.0649, 0.3493]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at McDonalds?\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3493]\", \"lift_point\": \"[0.0649, 0.3493]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at McDonalds?\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1227, 0.3625]\", \"lift_point\": \"[0.1227, 0.3625]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3493]\", \"lift_point\": \"[0.0649, 0.3493]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at McDonalds?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1227, 0.3625]\", \"lift_point\": \"[0.1227, 0.3625]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4172, 0.4398]\", \"lift_point\": \"[0.4172, 0.4398]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3493]\", \"lift_point\": \"[0.0649, 0.3493]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at McDonalds?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1227, 0.3625]\", \"lift_point\": \"[0.1227, 0.3625]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4172, 0.4398]\", \"lift_point\": \"[0.4172, 0.4398]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5612, 0.7968]\", \"lift_point\": \"[0.5612, 0.7968]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7938, 0.6776]\", \"lift_point\": \"[0.7938, 0.6776]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3493]\", \"lift_point\": \"[0.0649, 0.3493]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at McDonalds?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1227, 0.3625]\", \"lift_point\": \"[0.1227, 0.3625]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4172, 0.4398]\", \"lift_point\": \"[0.4172, 0.4398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5612, 0.7968]\", \"lift_point\": \"[0.5612, 0.7968]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6437, 0.93]\", \"lift_point\": \"[0.6437, 0.93]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7647, 0.6666]\", \"lift_point\": \"[0.7647, 0.6666]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7705, 0.6707]\", \"lift_point\": \"[0.7705, 0.6707]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0649, 0.3493]\", \"lift_point\": \"[0.0649, 0.3493]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats on the menu at McDonalds?\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1227, 0.3625]\", \"lift_point\": \"[0.1227, 0.3625]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.4172, 0.4398]\", \"lift_point\": \"[0.4172, 0.4398]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5612, 0.7968]\", \"lift_point\": \"[0.5612, 0.7968]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.6437, 0.93]\", \"lift_point\": \"[0.6437, 0.93]\", \"typed_text\": \"\" Goal: What's on the menu at McDonalds?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85070_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [PRESS_BACK,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.791, 0.6885]\", \"lift_point\": \"[0.791, 0.6885]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_10.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7909, 0.751]\", \"lift_point\": \"[0.7909, 0.751]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_11.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_12.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0664, 0.0473]\", \"lift_point\": \"[0.0664, 0.0473]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_13.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0675, 0.091]\", \"lift_point\": \"[0.0675, 0.091]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0755, 0.9017]\", \"lift_point\": \"[0.0755, 0.9017]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_14.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0673, 0.0607]\", \"lift_point\": \"[0.0673, 0.0607]\", \"typed_text\": \"\" \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0755, 0.9017]\", \"lift_point\": \"[0.0755, 0.9017]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0782, 0.9026]\", \"lift_point\": \"[0.0782, 0.9026]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_15.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_BACK\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0755, 0.9017]\", \"lift_point\": \"[0.0755, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0782, 0.9026]\", \"lift_point\": \"[0.0782, 0.9026]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9003]\", \"lift_point\": \"[0.0633, 0.9003]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_16.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0509, 0.915]\", \"lift_point\": \"[0.0509, 0.915]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0755, 0.9017]\", \"lift_point\": \"[0.0755, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0782, 0.9026]\", \"lift_point\": \"[0.0782, 0.9026]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9003]\", \"lift_point\": \"[0.0633, 0.9003]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9205]\", \"lift_point\": \"[0.0606, 0.9205]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_17.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0755, 0.9017]\", \"lift_point\": \"[0.0755, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0782, 0.9026]\", \"lift_point\": \"[0.0782, 0.9026]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9003]\", \"lift_point\": \"[0.0633, 0.9003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9205]\", \"lift_point\": \"[0.0606, 0.9205]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1799, 0.4773]\", \"lift_point\": \"[0.1799, 0.4773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_18.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0585, 0.9079]\", \"lift_point\": \"[0.0585, 0.9079]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0755, 0.9017]\", \"lift_point\": \"[0.0755, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0782, 0.9026]\", \"lift_point\": \"[0.0782, 0.9026]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9003]\", \"lift_point\": \"[0.0633, 0.9003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9205]\", \"lift_point\": \"[0.0606, 0.9205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1799, 0.4773]\", \"lift_point\": \"[0.1799, 0.4773]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1799, 0.4773]\", \"lift_point\": \"[0.1799, 0.4773]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_19.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0775, 0.9231]\", \"lift_point\": \"[0.0775, 0.9231]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0815, 0.9171]\", \"lift_point\": \"[0.0815, 0.9171]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0755, 0.9017]\", \"lift_point\": \"[0.0755, 0.9017]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0782, 0.9026]\", \"lift_point\": \"[0.0782, 0.9026]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0633, 0.9003]\", \"lift_point\": \"[0.0633, 0.9003]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0606, 0.9205]\", \"lift_point\": \"[0.0606, 0.9205]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1799, 0.4773]\", \"lift_point\": \"[0.1799, 0.4773]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1799, 0.4773]\", \"lift_point\": \"[0.1799, 0.4773]\", \"typed_text\": \"\" Goal: Open a new Chrome incognito tab>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85110_20.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Search for a new perfume>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4068]\", \"lift_point\": \"[0.8803, 0.4068]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4068]\", \"lift_point\": \"[0.8803, 0.4068]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4068]\", \"lift_point\": \"[0.8803, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.4162]\", \"lift_point\": \"[0.8923, 0.4162]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4068]\", \"lift_point\": \"[0.8803, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.4162]\", \"lift_point\": \"[0.8923, 0.4162]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3269]\", \"lift_point\": \"[0.0677, 0.3269]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4068]\", \"lift_point\": \"[0.8803, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.4162]\", \"lift_point\": \"[0.8923, 0.4162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3269]\", \"lift_point\": \"[0.0677, 0.3269]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new perfume\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_8.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4068]\", \"lift_point\": \"[0.8803, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.4162]\", \"lift_point\": \"[0.8923, 0.4162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3269]\", \"lift_point\": \"[0.0677, 0.3269]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new perfume\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.354, 0.392]\", \"lift_point\": \"[0.354, 0.392]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_9.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.774, 0.6812]\", \"lift_point\": \"[0.774, 0.6812]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.7713, 0.657]\", \"lift_point\": \"[0.7713, 0.657]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8803, 0.4068]\", \"lift_point\": \"[0.8803, 0.4068]\", \"typed_text\": \"\" \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8923, 0.4162]\", \"lift_point\": \"[0.8923, 0.4162]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0677, 0.3269]\", \"lift_point\": \"[0.0677, 0.3269]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"new perfume\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.354, 0.392]\", \"lift_point\": \"[0.354, 0.392]\", \"typed_text\": \"\" Goal: Search for a new perfume>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85180_10.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5211, 0.6851]\", \"lift_point\": \"[0.5211, 0.6851]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5211, 0.6851]\", \"lift_point\": \"[0.5211, 0.6851]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2579, 0.2913]\", \"lift_point\": \"[0.2579, 0.2913]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5211, 0.6851]\", \"lift_point\": \"[0.5211, 0.6851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2579, 0.2913]\", \"lift_point\": \"[0.2579, 0.2913]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.4715]\", \"lift_point\": \"[0.0817, 0.4715]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5211, 0.6851]\", \"lift_point\": \"[0.5211, 0.6851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2579, 0.2913]\", \"lift_point\": \"[0.2579, 0.2913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.4715]\", \"lift_point\": \"[0.0817, 0.4715]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bolivia\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5211, 0.6851]\", \"lift_point\": \"[0.5211, 0.6851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2579, 0.2913]\", \"lift_point\": \"[0.2579, 0.2913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.4715]\", \"lift_point\": \"[0.0817, 0.4715]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bolivia\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1624, 0.3986]\", \"lift_point\": \"[0.1624, 0.3986]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_6.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5211, 0.6851]\", \"lift_point\": \"[0.5211, 0.6851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2579, 0.2913]\", \"lift_point\": \"[0.2579, 0.2913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.4715]\", \"lift_point\": \"[0.0817, 0.4715]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1624, 0.3986]\", \"lift_point\": \"[0.1624, 0.3986]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3198, 0.3111]\", \"lift_point\": \"[0.3198, 0.3111]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_7.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8, 0.5]\", \"lift_point\": \"[0.2, 0.5]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.5211, 0.6851]\", \"lift_point\": \"[0.5211, 0.6851]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.2579, 0.2913]\", \"lift_point\": \"[0.2579, 0.2913]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0817, 0.4715]\", \"lift_point\": \"[0.0817, 0.4715]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"Whats the news in Bolivia\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1624, 0.3986]\", \"lift_point\": \"[0.1624, 0.3986]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.3198, 0.3111]\", \"lift_point\": \"[0.3198, 0.3111]\", \"typed_text\": \"\" Goal: What's the news in Bolivia?>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85209_8.png" + ] + }, + { + "query": "<\nPrevious Actions: Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [PRESS_HOME,DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85309_1.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3311]\", \"lift_point\": \"[0.8865, 0.3311]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85309_2.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3311]\", \"lift_point\": \"[0.8865, 0.3311]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.3478]\", \"lift_point\": \"[0.0653, 0.3478]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85309_3.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3311]\", \"lift_point\": \"[0.8865, 0.3311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.3478]\", \"lift_point\": \"[0.0653, 0.3478]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [TYPE,DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85309_4.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3311]\", \"lift_point\": \"[0.8865, 0.3311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.3478]\", \"lift_point\": \"[0.0653, 0.3478]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [DUAL_POINT,STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1408, 0.5501]\", \"lift_point\": \"[0.1408, 0.5501]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85309_5.png" + ] + }, + { + "query": "<\nPrevious Actions: \"action_type\": \"PRESS_HOME\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.8865, 0.3311]\", \"lift_point\": \"[0.8865, 0.3311]\", \"typed_text\": \"\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.0653, 0.3478]\", \"lift_point\": \"[0.0653, 0.3478]\", \"typed_text\": \"\" \"action_type\": \"TYPE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"capital of Paraguay\" \"action_type\": \"DUAL_POINT\", \"touch_point\": \"[0.1408, 0.5501]\", \"lift_point\": \"[0.1408, 0.5501]\", \"typed_text\": \"\" Goal: Google the capital of Paraguay>55555", + "response": "Action Plan: [STATUS_TASK_COMPLETE]\n; Action Decision: \"action_type\": \"STATUS_TASK_COMPLETE\", \"touch_point\": \"[-1.0, -1.0]\", \"lift_point\": \"[-1.0, -1.0]\", \"typed_text\": \"\"", + "images": [ + "/data/zbz5349/ICLR_2024/data/blip/general_texts_splits/85309_6.png" + ] + } +] \ No newline at end of file